def addSearchSpaceGrid(
hp: SearchSpaceGrid, disc: int, parent_disc: Hyperparameter, cs: ConfigurationSpace
) -> None:
smac = SearchSpaceGridtoSMAC(hp, disc)
for hyp in smac:
cs.add_hyperparameter(hyp)
cs.add_condition(EqualsCondition(child=hyp, parent=parent_disc, value=disc))
def optimize():
# We load the iris-dataset (a widely used benchmark)
iris = datasets.load_iris()
#logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
kernel = CategoricalHyperparameter("kernel", ["linear", "rbf", "poly", "sigmoid"], default="poly")
cs.add_hyperparameter(kernel)
# There are some hyperparameters shared by all kernels
C = UniformFloatHyperparameter("C", 0.001, 1000.0, default=1.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"], default="true")
cs.add_hyperparameters([C, shrinking])
# Others are kernel-specific, so we can add conditions to limit the searchspace
degree = UniformIntegerHyperparameter("degree", 1, 5, default=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0, default=0.0) # poly, sigmoid
cs.add_hyperparameters([degree, coef0])
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])
# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value", 0.0001, 8, default=1)
cs.add_hyperparameters([gamma, gamma_value])
# We only activate gamma_value if gamma is set to "value"
cs.add_condition(InCondition(child=gamma_value, parent=gamma, values=["value"]))
# And again we can restrict the use of gamma in general to the choice of the kernel
cs.add_condition(InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"]))
# Scenario object
scenario = Scenario("test/test_files/svm_scenario.txt")
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario, rng=np.random.RandomState(42),
tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
def main_loop(problem):
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
n_neighbors = UniformIntegerHyperparameter("n_neighbors", 2,10, default_value=5)
cs.add_hyperparameter(n_neighbors)
weights = CategoricalHyperparameter("weights", ["uniform","distance"], default_value="uniform")
algorithm = CategoricalHyperparameter("algorithm", ["ball_tree", "kd_tree","brute","auto"], default_value="auto")
cs.add_hyperparameters([weights, algorithm])
leaf_size = UniformIntegerHyperparameter("leaf_size", 1, 100, default_value=50)
cs.add_hyperparameter(leaf_size)
use_leaf_size= InCondition(child=leaf_size, parent=algorithm, values=["ball_tree","kd_tree"])
cs.add_condition(use_leaf_size)
p = UniformIntegerHyperparameter("p", 1,3, default_value=2)
cs.add_hyperparameter(p)
# Scenario object
max_eval=100000
scenario = Scenario({"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": max_eval, # maximum function evaluations
"cs": cs, # configuration space
"shared_model": True,
"output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/run_{}_{}_{}".format(max_eval,
datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S'),
problem)
# "output_dir": "/home/naamah/Documents/CatES/result_All/smac/KNN/{}/run_{}_{}".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')),
# "input_psmac_dirs":"/home/naamah/Documents/CatES/result_All/",
# "deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print("Optimizing! Depending on your machine, this might take a few minutes.")
smac = SMAC(scenario=scenario,tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
# main_loop()
def get_variables(problem):
cs = ConfigurationSpace()
params = []
for i in range(problem.dims()):
param = get_variable(problem, i)
params.append(param)
cs.add_hyperparameter(param)
for (i, c) in enumerate(problem.dependencies()):
if c is None:
continue
j = c['on']
s = c['values']
cs.add_condition(InCondition(params[i], params[j], list(s)))
return cs
def getPCS(self):
'''
maxIter: [1,100]最大迭代次数,默认50
regParam :[0,0.2] 正则化参数，默认0
tol:[1e-6,1e-1] 迭代算法收敛性，默认 1e-6
family ,link, variancePower 对应关系
• “gaussian” -> “identity”, “log”, “inverse”
• “binomial” -> “logit”, “probit”, “cloglog”
• “poisson” -> “log”, “identity”, “sqrt”
• “gamma” -> “inverse”, “identity”, “log”
• “tweedie” -> power link function specified through “linkPower”.
The default link power in the tweedie family is 1 - variancePower.
'''
# Build Configuration Space which defines all parameters and their
# ranges
cs = ConfigurationSpace()
maxIter = UniformIntegerHyperparameter("maxIter",
1,
100,
default_value=50)
regParam = UniformFloatHyperparameter("regParam",
0,
0.4,
default_value=1e-04)
tol = UniformFloatHyperparameter("tol",
1e-06,
1e-01,
default_value=1e-06)
family = CategoricalHyperparameter("family", ["gaussian", "poisson"],
default_value="gaussian")
gaussianLink = CategoricalHyperparameter(
"gaussianLink", ["identity", "log", "inverse"],
default_value="identity")
poissonLink = CategoricalHyperparameter("poissonLink",
["log", "identity", "sqrt"],
default_value="log")
cs.add_hyperparameters(
[maxIter, regParam, tol, family, gaussianLink, poissonLink])
cs.add_condition(
InCondition(child=gaussianLink, parent=family,
values=["gaussian"]))
cs.add_condition(
InCondition(child=poissonLink, parent=family, values=["poisson"]))
return cs
def build_config_space(clustering_ls=["KMeans", "DBSCAN"],
dim_reduction_ls=[]):
cs = ConfigurationSpace()
if len(clustering_ls) > 0:
clustering_choice = CategoricalHyperparameter(
"clustering_choice", clustering_ls, default_value=clustering_ls[0])
cs.add_hyperparameters([clustering_choice])
if len(dim_reduction_ls) > 0:
dim_reduction_choice = CategoricalHyperparameter(
"dim_reduction_choice",
dim_reduction_ls,
default_value=dim_reduction_ls[0])
cs.add_hyperparameters([dim_reduction_choice])
for idx, string in enumerate(
itertools.chain(clustering_ls, dim_reduction_ls)):
algorithm = Mapper.getClass(string)
# encode parameter names
encoded_params = []
for param in algorithm.params:
encoded_string = StringUtils.encode_parameter(
param.name, algorithm.name)
param.name = encoded_string
# add encoded paramters to configuration space
cs.add_hyperparameters(algorithm.params)
# define dependency
for param in algorithm.params:
cs.add_condition(
InCondition(child=param,
parent=clustering_choice if
idx < len(clustering_ls) else dim_reduction_choice,
values=[string]))
# add forbidden clauses
for condition in algorithm.forbidden_clauses:
cs.add_forbidden_clause(condition)
return cs
def build_config_space(clustering_ls: List[str] = None,
feature_selection_ls: List[str] = None):
if clustering_ls is None:
clustering_ls = ["KMeans", "DBSCAN"]
if feature_selection_ls is None:
feature_selection_ls = ["NullModel", "NormalizedCut"]
assert len(clustering_ls) > 0
assert len(feature_selection_ls) > 0
cs = ConfigurationSpace()
clustering_choice = CategoricalHyperparameter(
"clustering_choice", clustering_ls, default_value=clustering_ls[0])
feature_selection_choice = CategoricalHyperparameter(
"feature_selection_choice",
feature_selection_ls,
default_value=feature_selection_ls[0])
cs.add_hyperparameters([clustering_choice])
cs.add_hyperparameters([feature_selection_choice])
for idx, string in enumerate(
itertools.chain(clustering_ls, feature_selection_ls)):
algorithm = Mapper.get_class(string)
for param in algorithm.params:
encoded_string = encode_parameter(param.name, algorithm.name)
param.name = encoded_string
cs.add_hyperparameters(algorithm.params)
for param in algorithm.params:
cs.add_condition(
InCondition(
child=param,
parent=clustering_choice
if idx < len(clustering_ls) else feature_selection_choice,
values=[string]))
for condition in algorithm.forbidden_clauses:
cs.add_forbidden_clause(condition)
return cs
def config_space(self):
"""KNN hyperparameter space."""
n_neighbors = UniformIntegerHyperparameter(
'n_neighbors', lower=1, upper=100, default_value=5
)
leaf_size = UniformIntegerHyperparameter(
'leaf_size', lower=1, upper=100, default_value=20
)
metric = CategoricalHyperparameter(
'metric',
['euclidean', 'manhattan', 'chebyshev', 'minkowski'],
default_value='euclidean'
)
p_param = UniformIntegerHyperparameter('p', 1, 5, default_value=2)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters((n_neighbors, leaf_size, metric, p_param))
# Conditionals on hyperparameters specific to kernels.
config.add_condition(
InCondition(child=p_param, parent=metric, values=['minkowski'])
)
return config
def test_impute_inactive_hyperparameters(self):
cs = ConfigurationSpace()
a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1]))
b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1]))
c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1))
cs.add_condition(EqualsCondition(b, a, 1))
cs.add_condition(EqualsCondition(c, a, 0))
cs.seed(1)
configs = cs.sample_configuration(size=100)
config_array = convert_configurations_to_array(configs)
for line in config_array:
if line[0] == 0:
self.assertTrue(np.isnan(line[1]))
elif line[0] == 1:
self.assertTrue(np.isnan(line[2]))
gp = get_gp(3, np.random.RandomState(1))
config_array = gp._impute_inactive(config_array)
for line in config_array:
if line[0] == 0:
self.assertEqual(line[1], -1)
elif line[0] == 1:
self.assertEqual(line[2], -1)
def make_cs():
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("n_estimators", 1, 30, default=10))
max_features = CategoricalHyperparameter('max_features', ['auto', 'value'],
default='auto')
max_features_value = UniformFloatHyperparameter('max_features_value', .1,
1)
cs.add_hyperparameters([max_features, max_features_value])
cs.add_condition(
InCondition(child=max_features_value,
parent=max_features,
values=['value']))
max_depth = CategoricalHyperparameter('max_depth', [None, 'value'])
max_depth_value = UniformIntegerHyperparameter("max_depth_value", 1, 10)
cs.add_hyperparameters([max_depth, max_depth_value])
cs.add_condition(
InCondition(child=max_depth_value, parent=max_depth, values=['value']))
min_samples_split = UniformFloatHyperparameter("min_samples_split", .1, 1)
cs.add_hyperparameter(min_samples_split)
min_samples_leaf = UniformFloatHyperparameter("min_samples_leaf", .1, .5)
cs.add_hyperparameter(min_samples_leaf)
min_weight_fraction_leaf = UniformFloatHyperparameter(
"min_weight_fraction_leaf", 0, .5)
cs.add_hyperparameter(min_weight_fraction_leaf)
max_leaf_nodes = CategoricalHyperparameter('max_leaf_nodes',
[None, 'value'])
max_leaf_nodes_value = UniformIntegerHyperparameter(
'max_leaf_nodes_value', 2, 100)
cs.add_hyperparameters([max_leaf_nodes, max_leaf_nodes_value])
cs.add_condition(
InCondition(child=max_leaf_nodes_value,
parent=max_leaf_nodes,
values=['value']))
min_impurity_split = UniformFloatHyperparameter('min_impurity_split', 0, 1)
cs.add_hyperparameter(min_impurity_split)
bootstrap = CategoricalHyperparameter('bootstrap', [True, False],
default=True)
cs.add_hyperparameter(bootstrap)
class SMACOptimizer(PhotonBaseOptimizer):
def __init__(self,
run_obj="quality",
wallclock_limit: float = float("inf"),
min_r=1,
max_r=1,
run_limit=5,
smac_helper=None
): # optimizer only for test and developing settings yet.
if not __found__:
raise ModuleNotFoundError(
"Module smac not found or not installed as expected. "
"Please install the smac_requirements.txt PHOTON provides.")
self.wallclock_limit = wallclock_limit
self.minR = min_r
self.maxR = max_r
if self.minR != 1 or self.maxR != 1:
raise NotImplementedError(
"PHOTONs seed management is not implemented yet. "
"At this juncture we can not run a config multiple times correctly."
)
self.cspace = ConfigurationSpace() # Hyperparameter space for smac
self.runtime = 0
self.run_start = 0
self.run_obj = run_obj
self.run_limit = run_limit
self.challengers = []
self.old_challengers = None
self.ask_list = []
self.switch_optiones = {}
self.hyperparameters = []
self.budget_exhausted = False
self.constant_dictionary = {}
self.smac_helper = smac_helper
self.maximize_metric = False
@staticmethod
def _convert_photon_to_smac_space(hyperparam: object, name: str):
"""
Helper function: Convert PHOTON hyperparams to smac params.
"""
if not hyperparam:
return None
if isinstance(hyperparam, PhotonCategorical):
return CategoricalHyperparameter(name, hyperparam.values)
elif isinstance(hyperparam, list):
return CategoricalHyperparameter(name, hyperparam)
elif isinstance(hyperparam, FloatRange):
if hyperparam.range_type == 'linspace':
return UniformFloatHyperparameter(name, hyperparam.start,
hyperparam.stop)
elif hyperparam.range_type == 'logspace':
raise NotImplementedError(
"Logspace in your float hyperparameter is not implemented in SMAC."
)
else:
return UniformFloatHyperparameter(name, hyperparam.start,
hyperparam.stop)
elif isinstance(hyperparam, IntegerRange):
return UniformIntegerHyperparameter(name, hyperparam.start,
hyperparam.stop)
def build_smac_space(self, pipeline_elements):
"""
Build entire smac hyperparam space.
"""
for pipe_element in pipeline_elements:
# build conditions for switch elements
# maybe this creepy condition is so that we avoid circular imports!
if pipe_element.__class__.__name__ == 'Switch':
algorithm_options = {}
for algo in pipe_element.elements:
algo_params = [] # hyper params corresponding to "algo"
for name, value in algo.hyperparameters.items():
smac_param = self._convert_photon_to_smac_space(
value,
(pipe_element.name + "__" +
name)) # or element.name__algo.name__name ???
algo_params.append(smac_param)
algorithm_options[(pipe_element.name + "__" +
algo.name)] = algo_params
algos = CategoricalHyperparameter(
name=pipe_element.name + "__algos",
choices=algorithm_options.keys())
self.switch_optiones[pipe_element.name +
"__algos"] = algorithm_options.keys()
self.cspace.add_hyperparameter(algos)
for algo, params in algorithm_options.items():
for param in params:
cond = InCondition(child=param,
parent=algos,
values=[algo])
self.cspace.add_hyperparameter(param)
self.cspace.add_condition(cond)
continue
if hasattr(pipe_element, 'hyperparameters'):
for name, value in pipe_element.hyperparameters.items():
self.hyperparameters.append(name)
# if we only have one value we do not need to optimize
if isinstance(value, list) and len(value) < 2:
self.constant_dictionary[name] = value[0]
continue
if isinstance(value,
PhotonCategorical) and len(value.values) < 2:
self.constant_dictionary[name] = value.values[0]
continue
smac_param = self._convert_photon_to_smac_space(
value, name)
if smac_param is not None:
self.cspace.add_hyperparameter(smac_param)
def prepare(self, pipeline_elements: list, maximize_metric: bool):
# build space
self.space = ConfigurationSpace()
self.build_smac_space(pipeline_elements)
self.maximize_metric = maximize_metric
self.scenario = Scenario({
"run_obj": self.run_obj,
"cs": self.cspace,
"deterministic": "true",
"wallclock_limit": self.wallclock_limit
})
self.smac = SMAC4BO(scenario=self.scenario,
rng=np.random.RandomState(42),
tae_runner=MyExecuteTARun)
if self.smac_helper:
self.smac_helper["data"] = self.smac
self.optimizer = self.smac.solver
self.optimizer.runhistory.overwrite_existing_runs = True
self.ask = self.ask_generator()
def tell(self, config, performance):
if not config:
return
config = copy.copy(config)
for key in self.switch_optiones.keys():
config[key] = [
x for x in self.switch_optiones[key]
if any(x in val for val in config.keys())
][0]
if self.maximize_metric:
performance = 1 - performance[1]
config = Configuration(self.cspace, values=config)
self.optimizer.stats.ta_runs += 1
# first incubment setting
if self.runtime == 0:
self.optimizer.incumbent = Configuration(
self.cspace, values=config) #self.challengers[0])
self.optimizer.start()
else:
if self.optimizer.stats.is_budget_exhausted():
self.budget_exhausted = True
self.optimizer.runhistory.add(config=config,
cost=performance,
time=0,
status=StatusType.SUCCESS,
seed=0)
self.runtime += 1
def ask_generator(self):
def init():
if self.runtime == 0:
ta_run = MyExecuteTARun(run_limit=self.run_limit,
runhistory=self.optimizer.runhistory)
self.optimizer.intensifier = Intensifier(
tae_runner=ta_run,
stats=self.optimizer.stats,
traj_logger=None,
rng=np.random.RandomState(42),
instances=self.scenario.
train_insts, #list(self.optimizer.runhistory.ids_config.keys()),
minR=1,
maxR=1,
min_chall=self.scenario.intens_min_chall,
instance_specifics=self.scenario.instance_specific) # ,
# run_limit=self.run_limit)
#self.optimizer.stats.start_timing()
self.old_challengers = self.optimizer.initial_design.select_configurations(
)
#a = self.optimizer.initial_design.select_configurations()[0].get_dictionary()
#tmp_dict = x.get_dictionary()
self.ask_list = [{
key: x.get_dictionary()[key]
for key in x.get_dictionary().keys() if 'algos' not in key
} for x in self.optimizer.initial_design.select_configurations(
)]
self.smac_helper = len(self.ask_list)
print("-----SMAC INIT READY----")
else:
start_time = time.time()
X, Y = self.optimizer.rh2EPM.transform(
self.optimizer.runhistory)
self.optimizer.logger.debug("Search for next configuration")
# get all found configurations sorted according to acq
challengers = self.optimizer.choose_next(X, Y)
time_spent = time.time() - start_time
time_left = self.optimizer._get_timebound_for_intensification(
time_spent)
# first run challenger vs. any other run
if isinstance(challengers, list):
self.challengers = challengers[:self.run_limit]
else:
self.challengers = challengers.challengers[:self.run_limit]
self.ask_list = [{
key: x.get_dictionary()[key]
for key in x.get_dictionary().keys() if 'algos' not in key
} for x in self.challengers]
if self.old_challengers:
self.incumbent, inc_perf = self.optimizer.intensifier.\
intensify(challengers=self.old_challengers,
incumbent=self.optimizer.incumbent,
run_history=self.optimizer.runhistory,
aggregate_func=self.optimizer.aggregate_func,
log_traj=False,
time_bound=max(self.optimizer.intensifier._min_time, time_left))
self.optimizer.incumbent = self.incumbent
self.old_challengers = self.challengers
return 0
i = init()
while True:
if self.budget_exhausted:
val = yield {}
return
else:
val = (yield self.ask_list[i])
if len(self.ask_list) - 1 == i:
i = init()
else:
i += 1
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])
# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter(
"gamma", ["auto", "value"],
default_value="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value",
0.0001,
8,
default_value=1)
cs.add_hyperparameters([gamma, gamma_value])
# We only activate gamma_value if gamma is set to "value"
cs.add_condition(InCondition(child=gamma_value, parent=gamma,
values=["value"]))
# And again we can restrict the use of gamma in general to the choice of the kernel
cs.add_condition(
InCondition(child=gamma, parent=kernel, values=["rbf", "poly", "sigmoid"]))
# Scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": 200, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
def cs_single():
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
root = CategoricalHyperparameter("root", choices=["l1", "ln"])
x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"])
x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"])
x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"])
x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"])
x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"])
x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"])
# r1 is the data associated in x1
r1 = UniformFloatHyperparameter("r1", lower=0.01, upper=0.99, log=False)
r2 = UniformFloatHyperparameter("r2", lower=0.01, upper=0.99, log=False)
r3 = UniformFloatHyperparameter("r3", lower=0.01, upper=0.99, log=False)
r4 = UniformFloatHyperparameter("r4", lower=0.01, upper=0.99, log=False)
r5 = UniformFloatHyperparameter("r5", lower=0.01, upper=0.99, log=False)
r6 = UniformFloatHyperparameter("r6", lower=0.01, upper=0.99, log=False)
r7 = UniformFloatHyperparameter("r7", lower=0.01, upper=0.99, log=False)
r8 = UniformFloatHyperparameter("r8", lower=0.01, upper=0.99, log=False)
r9 = UniformFloatHyperparameter("r9", lower=0.01, upper=0.99, log=False)
r10 = UniformFloatHyperparameter("r10", lower=0.01, upper=0.99, log=False)
r11 = UniformFloatHyperparameter("r11", lower=0.01, upper=0.99, log=False)
r12 = UniformFloatHyperparameter("r12", lower=0.01, upper=0.99, log=False)
r13 = UniformFloatHyperparameter("r13", lower=0.01, upper=0.99, log=False)
r14 = UniformFloatHyperparameter("r14", lower=0.01, upper=0.99, log=False)
cs.add_hyperparameters([
root,
x1,
x2,
x3,
x4,
x5,
x6,
r1,
r2,
r3,
r4,
r5,
r6,
r7,
r8,
r9,
r10,
r11,
r12,
r13,
r14,
])
# add condition
cs.add_condition(InCondition(x1, root, ["l1"]))
cs.add_condition(InCondition(x2, root, ["ln"]))
cs.add_condition(InCondition(r1, root, ["l1"]))
cs.add_condition(InCondition(r2, root, ["ln"]))
cs.add_condition(InCondition(x3, x1, ["l1"]))
cs.add_condition(InCondition(x4, x1, ["ln"]))
cs.add_condition(InCondition(r3, x1, ["l1"]))
cs.add_condition(InCondition(r4, x1, ["ln"]))
cs.add_condition(InCondition(x5, x2, ["l1"]))
cs.add_condition(InCondition(x6, x2, ["ln"]))
cs.add_condition(InCondition(r5, x2, ["l1"]))
cs.add_condition(InCondition(r6, x2, ["ln"]))
cs.add_condition(InCondition(r7, x3, ["l1"]))
cs.add_condition(InCondition(r8, x3, ["ln"]))
cs.add_condition(InCondition(r9, x4, ["l1"]))
cs.add_condition(InCondition(r10, x4, ["ln"]))
cs.add_condition(InCondition(r11, x5, ["l1"]))
cs.add_condition(InCondition(r12, x5, ["ln"]))
cs.add_condition(InCondition(r13, x6, ["l1"]))
cs.add_condition(InCondition(r14, x6, ["ln"]))
return cs
def bayesianmcmc(self):
from sklearn.decomposition import PCA
from sklearn.manifold import Isomap
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from mahotas.features import haralick
import os
from sklearn.model_selection import StratifiedKFold, train_test_split
from sklearn import metrics
from sklearn.preprocessing import StandardScaler
import cv2
from sklearn.neighbors import KNeighborsClassifier
def naive_all_features(names):
f = []
for i in range(len(names)):
I = cv2.imread(names[i])
l = I.shape
f1 = []
if I is None or I.size == 0 or np.sum(
I[:]) == 0 or I.shape[0] == 0 or I.shape[1] == 0:
if len(l) == 3:
f1 = np.zeros((1, l[0] * l[1] * l[2]))
else:
f1 = I.flatten()
f1 = np.expand_dims(f1, 0)
if i == 0:
f = f1
else:
f = np.vstack((f, f1))
return f
def haralick_all_features(X, distance=1):
f = []
for i in range(len(X)):
I = cv2.imread(X[i])
if I is None or I.size == 0 or np.sum(
I[:]) == 0 or I.shape[0] == 0 or I.shape[1] == 0:
h = np.zeros((1, 13))
else:
I = cv2.cvtColor(I, cv2.COLOR_BGR2GRAY)
h = haralick(I,
distance=distance,
return_mean=True,
ignore_zeros=False)
h = np.expand_dims(h, 0)
if i == 0:
f = h
else:
f = np.vstack((f, h))
return f
def CNN_all_features(names, cnn):
from keras.applications.vgg19 import VGG19
from keras.applications.inception_v3 import InceptionV3
from keras.applications.vgg19 import preprocess_input
f = []
if cnn == 'VGG':
model = VGG19(weights='imagenet')
dsize = (224, 224)
else:
model = InceptionV3(weights='imagenet')
dsize = (299, 299)
for i in range(len(names)):
img = cv2.imread(names[i])
img = cv2.resize(img, dsize=dsize)
img = img.astype('float32')
x = np.expand_dims(img, axis=0)
x = preprocess_input(x)
features = model.predict(x)
if i == 0:
f = features
else:
f = np.vstack((f, features))
return f
def VGG_all_features(names, X):
home = os.path.expanduser('~')
if os.path.exists(self.data_loc + 'features/bayesian1/VGG_' +
self.data_name + '.npz'):
f = np.load(
open(
self.data_loc + 'features/bayesian1/VGG_' +
self.data_name + '.npz', 'rb'))
return f.f.arr_0[X, :]
else:
f = CNN_all_features(names, 'VGG')
np.savez(
open(
self.data_loc + 'features/bayesian1/VGG_' +
self.data_name + '.npz', 'wb'), f)
return f[X, :]
def inception_all_features(names, X):
home = os.path.expanduser('~')
if os.path.exists(self.data_loc + 'features/bayesian1/inception_' +
self.data_name + '.npz'):
f = np.load(
open(
self.data_loc + 'features/bayesian1/inception_' +
self.data_name + '.npz', 'rb'))
return f.f.arr_0[X, :]
else:
f = CNN_all_features(names, 'inception')
np.savez(
open(
self.data_loc + 'features/bayesian1/inception_' +
self.data_name + '.npz', 'wb'), f)
return f[X, :]
def principal_components(X, whiten=True):
pca = PCA(whiten=whiten)
maxvar = 0.95
X = np.asarray(X)
if len(X.shape) == 1:
X = X.reshape(-1, 1)
data = X
X1 = pca.fit(X)
var = pca.explained_variance_ratio_
s1 = 0
for i in range(len(var)):
s1 += var[i]
s = 0
for i in range(len(var)):
s += var[i]
if (s * 1.0 / s1) >= maxvar:
break
pca = PCA(n_components=i + 1)
pca.fit(data)
return pca
def isomap(X, n_neighbors=5, n_components=2):
iso = Isomap(n_components=n_components, n_neighbors=n_neighbors)
X = np.asarray(X)
if len(X.shape) == 1:
X = X.reshape(-1, 1)
iso.fit(X)
return iso
def random_forests(X, y, n_estimators, max_features):
clf = RandomForestClassifier(n_estimators=n_estimators,
max_features=max_features,
class_weight='balanced')
clf.fit(X, y)
return clf
def support_vector_machines(X, y, C, gamma):
clf = svm.SVC(C=C,
gamma=gamma,
class_weight='balanced',
probability=True)
clf.fit(X, y)
return clf
def knn(X, y, neighbors=1):
clf = KNeighborsClassifier(n_neighbors=neighbors)
clf.fit(X, y)
return clf
def pipeline_from_cfg(cfg):
cfg = {k: cfg[k] for k in cfg if cfg[k]}
# Load the data
data_home = self.data_loc + 'datasets/' + self.data_name + '/'
l1 = os.listdir(data_home)
y = []
names = []
cnt = 0
for z in range(len(l1)):
if l1[z][0] == '.':
continue
l = os.listdir(data_home + l1[z] + '/')
y += [cnt] * len(l)
cnt += 1
for i in range(len(l)):
names.append(data_home + l1[z] + '/' + l[i])
# Train val split
X = np.empty((len(y), 1))
indices = np.arange(len(y))
X1, _, y1, y_val, id1, _ = train_test_split(X,
y,
indices,
test_size=0.2,
random_state=42,
shuffle=True)
s = []
val_splits = 3
kf = StratifiedKFold(n_splits=val_splits,
random_state=42,
shuffle=True)
names1 = []
for i in range(len(id1)):
names1.append((names[id1[i]]))
f11 = []
for idx1, idx2 in kf.split(X1, y1):
# Feature extraction
ids1 = []
X_train = []
y_train = []
for i in idx1:
X_train.append(names1[i])
y_train.append(y1[i])
ids1.append(id1[i])
X_val = []
y_val = []
ids2 = []
for i in idx2:
X_val.append(names1[i])
y_val.append(y1[i])
ids2.append(id1[i])
# Feature extraction
f_train = []
# f_test = []
f_val = []
if cfg['feature_extraction'] == "haralick":
f_val = haralick_all_features(X_val,
cfg["haralick_distance"])
f_train = haralick_all_features(X_train,
cfg["haralick_distance"])
elif cfg['feature_extraction'] == "VGG":
f_val = VGG_all_features(names, ids2)
f_train = VGG_all_features(names, ids1)
elif cfg['feature_extraction'] == "inception":
f_val = inception_all_features(names, ids2)
f_train = inception_all_features(names, ids1)
elif cfg['feature_extraction'] == "naive_feature_extraction":
f_val = naive_all_features(X_val)
f_train = naive_all_features(X_train)
# Dimensionality reduction
if cfg['dimensionality_reduction'] == "PCA":
cfg["pca_whiten"] = True if cfg[
"pca_whiten"] == "true" else False
dr = principal_components(f_train, cfg["pca_whiten"])
f_train = dr.transform(f_train)
f_val = dr.transform(f_val)
elif cfg['dimensionality_reduction'] == "ISOMAP":
dr = isomap(f_train, cfg["n_neighbors"],
cfg["n_components"])
f_train = dr.transform(f_train)
f_val = dr.transform(f_val)
elif cfg[
'dimensionality_reduction'] == 'naive_dimensionality_reduction':
f_train = f_train
f_val = f_val
# Pre-processing
normalizer = StandardScaler().fit(f_train)
f_train = normalizer.transform(f_train)
f_val = normalizer.transform(f_val)
# Learning algorithms
if cfg['learning_algorithm'] == "RF":
clf = random_forests(f_train, y_train, cfg["n_estimators"],
cfg["max_features"])
elif cfg['learning_algorithm'] == "SVM":
clf = support_vector_machines(f_train, y_train,
cfg["svm_C"],
cfg["svm_gamma"])
elif cfg['learning_algorithm'] == 'naive_learning_algorithm':
clf = knn(f_train, y_train)
p_pred = clf.predict_proba(f_val)
f11.append(metrics.log_loss(y_val, p_pred))
s.append(clf.score(f_val, y_val))
return np.mean(f11)
self.potential = []
self.best_pipelines = []
self.times = []
self.error_curves = []
cs = ConfigurationSpace()
feature_extraction = CategoricalHyperparameter(
"feature_extraction", ["haralick", "VGG", "inception"],
default="haralick")
cs.add_hyperparameter(feature_extraction)
dimensionality_reduction = CategoricalHyperparameter(
"dimensionality_reduction", ["PCA", "ISOMAP"], default="PCA")
cs.add_hyperparameter(dimensionality_reduction)
learning_algorithm = CategoricalHyperparameter("learning_algorithm",
["SVM", "RF"],
default="RF")
cs.add_hyperparameter(learning_algorithm)
haralick_distance = UniformIntegerHyperparameter("haralick_distance",
1,
3,
default=1)
cs.add_hyperparameter(haralick_distance)
cond1 = InCondition(child=haralick_distance,
parent=feature_extraction,
values=["haralick"])
cs.add_condition(cond1)
pca_whiten = CategoricalHyperparameter("pca_whiten", ["true", "false"],
default="true")
cs.add_hyperparameter(pca_whiten)
cs.add_condition(
InCondition(child=pca_whiten,
parent=dimensionality_reduction,
values=["PCA"]))
n_neighbors = UniformIntegerHyperparameter("n_neighbors",
3,
7,
default=5)
n_components = UniformIntegerHyperparameter("n_components",
2,
4,
default=2)
cs.add_hyperparameters([n_neighbors, n_components])
cs.add_condition(
InCondition(child=n_components,
parent=dimensionality_reduction,
values=["ISOMAP"]))
cs.add_condition(
InCondition(child=n_neighbors,
parent=dimensionality_reduction,
values=["ISOMAP"]))
svm_C = UniformFloatHyperparameter("svm_C", 0.1, 100.0, default=1.0)
cs.add_hyperparameter(svm_C)
svm_gamma = UniformFloatHyperparameter("svm_gamma", 0.01, 8, default=1)
cs.add_hyperparameter(svm_gamma)
cond1 = InCondition(child=svm_C,
parent=learning_algorithm,
values=["SVM"])
cond2 = InCondition(child=svm_gamma,
parent=learning_algorithm,
values=["SVM"])
cs.add_conditions([cond1, cond2])
n_estimators = UniformIntegerHyperparameter("n_estimators",
8,
300,
default=10)
max_features = UniformFloatHyperparameter("max_features",
0.3,
0.8,
default=0.5)
cs.add_hyperparameters([max_features, n_estimators])
cond1 = InCondition(child=n_estimators,
parent=learning_algorithm,
values=["RF"])
cond2 = InCondition(child=max_features,
parent=learning_algorithm,
values=["RF"])
cs.add_conditions([cond1, cond2])
scenario = Scenario({
"run_obj": "quality",
"cutoff_time": 100000,
"runcount_limit": 10000 * 10,
"cs": cs,
"maxR": 100000,
"wallclock_limit": 1000000,
"deterministic": "true"
})
smac = SMAC(scenario=scenario,
rng=np.random.RandomState(42),
tae_runner=pipeline_from_cfg)
incumbent, incs, incumbents, incumbents1, times = smac.optimize()
inc_value = pipeline_from_cfg(incumbent)
self.best_pipelines.append(incumbent)
self.potential.append(inc_value)
self.incumbents = incumbents
self.all_incumbents = incumbents1
self.error_curves.append(incs)
self.times = times
pickle.dump(
self,
open(
self.results_loc + 'intermediate/SMAC/SMAC_' + self.data_name +
'_run_' + str(self.run) + '_full.pkl', 'wb'))
class SMACOptimizer(PhotonBaseOptimizer):
def __init__(
self,
cutoff_time: float = float("inf"),
run_obj: str = "quality",
runcount_limit: float = float("inf"),
tuner_timeout: float = float("inf"),
wallclock_limit: float = float("inf"),
):
self.optimizer = None
self.scenario = None
self.hyperparameter_list = []
self.metric_to_optimize = ""
if (
run_obj == "runtime" and cutoff_time == float("inf")
) or run_obj == "quality":
cutoff_time = None
self.ask = self.ask_generator()
self.runcount_limit = (
runcount_limit
) # should there be something more? Refer - https://automl.github.io/SMAC3/stable/options.html
self.tuner_timeout = tuner_timeout
self.wallclock_limit = wallclock_limit
self.run_obj = run_obj
self.cutoff_time = cutoff_time
# self.maximize_metric = True
self.memory = {} # stores config : cost
self.test = 0
def prepare(self, pipeline_elements: list, maximize_metric: bool):
self.hyperparameter_list = []
# build space
self.space = ConfigurationSpace()
for element in pipeline_elements:
# check if Switch object
if isinstance(element, Switch):
algorithm_options = (
{}
) # mapping algorithm name with their child hyper params
for algo in element.elements:
algo_params = [] # hyper params corresponding to "algo"
for name, value in algo.hyperparameters.items():
smac_param = self._convert_PHOTON_to_smac_space(
value, (element.name + "__" + name)
) # or element.name__algo.name__name ???
algo_params.append(smac_param)
algorithm_options[(element.name + "__" + algo.name)] = algo_params
algos = CategoricalHyperparameter(
name=element.name + "__algos", choices=algorithm_options.keys()
)
self.space.add_hyperparameter(algos)
for algo, params in algorithm_options.items():
for param in params:
cond = InCondition(child=param, parent=algos, values=[algo])
self.space.add_hyperparameter(param)
self.space.add_condition(cond)
else:
for name, value in element.hyperparameters.items():
smac_param = self._convert_PHOTON_to_smac_space(value, name)
if smac_param is not None:
self.space.add_hyperparameter(smac_param)
self.scenario = Scenario(
{
"run_obj": self.run_obj,
"cutoff_time": self.cutoff_time,
"runcount_limit": self.runcount_limit,
"tuner-timeout": self.tuner_timeout,
"wallclock_limit": self.wallclock_limit,
"cs": self.space,
"deterministic": "true",
}
)
self.smac = SMAC4BO(scenario=self.scenario, rng=np.random.RandomState(42))
self.optimizer = self.smac.solver
self.optimizer.stats.start_timing()
self.optimizer.incumbent = self.get_default_incumbent()
self.flag = (
False
) # False: compute performance of challenger, True: compute performance of incumbent
self.ask = self.ask_generator()
def get_default_incumbent(self):
instantiated_hyperparameters = {}
for hp in self.space.get_hyperparameters():
conditions = self.space._get_parent_conditions_of(hp.name)
active = True
for condition in conditions:
parent_names = [
c.parent.name for c in condition.get_descendant_literal_conditions()
]
parents = {
parent_name: instantiated_hyperparameters[parent_name]
for parent_name in parent_names
}
if not condition.evaluate(parents):
# TODO find out why a configuration is illegal!
active = False
if not active:
instantiated_hyperparameters[hp.name] = None
elif isinstance(hp, Constant):
instantiated_hyperparameters[hp.name] = hp.value
else:
instantiated_hyperparameters[hp.name] = hp.default_value
config = Configuration(self.space, instantiated_hyperparameters)
return config
def _convert_PHOTON_to_smac_space(self, hyperparam: object, name: str):
if not hyperparam:
return None
self.hyperparameter_list.append(name)
if isinstance(hyperparam, PhotonCategorical):
return CategoricalHyperparameter(choices=hyperparam.values, name=name)
elif isinstance(hyperparam, list):
return CategoricalHyperparameter(choices=hyperparam, name=name)
elif isinstance(hyperparam, FloatRange):
return UniformFloatHyperparameter(
lower=hyperparam.start, upper=hyperparam.stop, name=name
)
elif isinstance(hyperparam, IntegerRange):
return UniformIntegerHyperparameter(
lower=hyperparam.start, upper=hyperparam.stop, name=name
)
elif isinstance(hyperparam, BooleanSwitch):
return CategoricalHyperparameter(choices=["true", "false"], name=name)
def ask_generator(self):
while True:
self.flag = False
start_time = time.time()
X, Y = self.optimizer.rh2EPM.transform(self.optimizer.runhistory)
self.optimizer.logger.debug("Search for next configuration.")
# get all configurations sorted according to acquision function
challengers = self.optimizer.choose_next(X, Y)
self.test += 1
print("TEST # of trains", self.test)
time_spent = time.time() - start_time
time_left = self.optimizer._get_timebound_for_intensification(time_spent)
self.to_run = self.intensify(
challengers=challengers,
incumbent=self.optimizer.incumbent,
run_history=self.optimizer.runhistory,
aggregate_func=self.optimizer.aggregate_func,
time_bound=max(self.optimizer.intensifier._min_time, time_left),
)
if self.flag:
if self.optimizer.stats.is_budget_exhausted():
# yield self.optimizer.incumbent.get_dictionary()
return None
else:
yield self.check(self.to_run.get_dictionary())
else:
print("Size of challenger list: ", len(self.to_run))
for challenger in self.to_run[: min(len(self.to_run), 25)]:
if self.optimizer.stats.is_budget_exhausted():
# yield self.optimizer.incumbent.get_dictionary()
return None
else:
yield self.check(challenger.get_dictionary())
logger.debug(
"Remaining budget: %f (wallclock), %f (ta costs), %f (target runs)"
% (
self.optimizer.stats.get_remaing_time_budget(),
self.optimizer.stats.get_remaining_ta_budget(),
self.optimizer.stats.get_remaining_ta_runs(),
)
)
self.optimizer.stats.print_stats(debug_out=True)
def check(self, config):
config_dict = {}
self.flag0 = False
for k, val in config.items():
if not "algos" in k:
config_dict[k] = val
else:
self.temp = (k, val)
self.flag0 = True
return config_dict
def intensify(
self, challengers, incumbent, run_history, aggregate_func, time_bound
):
instance_specifics = {}
_num_run = 0
_chall_indx = 0
_ta_time = 0
_min_time = 10 ** -5
start_time = time.time()
to_run = None
if time_bound < _min_time:
raise ValueError("time_bound must be >= %f" % (_min_time))
to_run_configs = []
for challenger in challengers:
if challenger == incumbent:
continue
if self.hashable_dict(incumbent.get_dictionary()) not in self.memory:
self.flag = True
return incumbent
to_run = self.race_challenger(challenger=challenger)
if to_run is not None:
to_run_configs.append(to_run)
return to_run_configs
def race_challenger(self, challenger):
if self.hashable_dict(challenger.get_dictionary()) not in self.memory:
return challenger
else:
return None
def compare_configs(self, incumbent, challenger):
inc_perf = self.memory[self.hashable_dict(incumbent.get_dictionary())]
chal_perf = self.memory[self.hashable_dict(challenger.get_dictionary())]
if inc_perf > chal_perf:
return incumbent
else:
return challenger
def hashable_dict(self, config: dict):
return frozenset(config.items())
def tell(self, config: dict, performance: float, runtime: float = 2.0):
if self.flag0:
config[self.temp[0]] = self.temp[1]
performance = performance[1]
self.optimizer.stats.ta_runs += 1
self.optimizer.stats.ta_time_used += runtime
hash_config = self.hashable_dict(config)
self.memory[hash_config] = performance
# convert dictionary to list in correct order
config = Configuration(self.space, values=config)
if self.run_obj == "runtime":
performance = -runtime
if runtime > self.cutoff_time:
runtime = self.cutoff_time
self.optimizer.runhistory.add(
config=config,
cost=performance,
time=runtime,
status=StatusType.TIMEOUT,
)
else:
self.optimizer.runhistory.add(
config=config,
cost=performance,
time=runtime,
status=StatusType.SUCCESS,
)
else:
self.optimizer.runhistory.add(
config=config, cost=performance, time=runtime, status=StatusType.SUCCESS
)
if self.flag:
pass
else:
self.optimizer.incumbent = self.compare_configs(
self.optimizer.incumbent, config
)
def get_cs():
cs = ConfigurationSpace()
root = CategoricalHyperparameter("root", choices=["l1", "ln"])
x1 = CategoricalHyperparameter("x1", choices=["l1", "ln"])
x2 = CategoricalHyperparameter("x2", choices=["l1", "ln"])
x3 = CategoricalHyperparameter("x3", choices=["l1", "ln"])
x4 = CategoricalHyperparameter("x4", choices=["l1", "ln"])
x5 = CategoricalHyperparameter("x5", choices=["l1", "ln"])
x6 = CategoricalHyperparameter("x6", choices=["l1", "ln"])
# r1 is the data associated in x1
r1_1 = UniformFloatHyperparameter("r1_1",
lower=0.01,
upper=0.99,
log=False)
r1_2 = UniformFloatHyperparameter("r1_2",
lower=0.01,
upper=0.99,
log=False)
r1_3 = UniformFloatHyperparameter("r1_3",
lower=0.01,
upper=0.99,
log=False)
r1_4 = UniformFloatHyperparameter("r1_4",
lower=0.01,
upper=0.99,
log=False)
r2_1 = UniformFloatHyperparameter("r2_1",
lower=0.01,
upper=0.99,
log=False)
r2_2 = UniformFloatHyperparameter("r2_2",
lower=0.01,
upper=0.99,
log=False)
r2_3 = UniformFloatHyperparameter("r2_3",
lower=0.01,
upper=0.99,
log=False)
r2_4 = UniformFloatHyperparameter("r2_4",
lower=0.01,
upper=0.99,
log=False)
r3_1 = UniformFloatHyperparameter("r3_1",
lower=0.01,
upper=0.99,
log=False)
r3_2 = UniformFloatHyperparameter("r3_2",
lower=0.01,
upper=0.99,
log=False)
r3_3 = Constant("r3_3", 0.5)
r3_4 = Constant("r3_4", 0.5)
r4_1 = UniformFloatHyperparameter("r4_1",
lower=0.01,
upper=0.99,
log=False)
r4_2 = UniformFloatHyperparameter("r4_2",
lower=0.01,
upper=0.99,
log=False)
r4_3 = Constant("r4_3", 0.5)
r4_4 = Constant("r4_4", 0.5)
r5_1 = UniformFloatHyperparameter("r5_1",
lower=0.01,
upper=0.99,
log=False)
r5_2 = UniformFloatHyperparameter("r5_2",
lower=0.01,
upper=0.99,
log=False)
r5_3 = Constant("r5_3", 0.5)
r5_4 = Constant("r5_4", 0.5)
r6_1 = UniformFloatHyperparameter("r6_1",
lower=0.01,
upper=0.99,
log=False)
r6_2 = UniformFloatHyperparameter("r6_2",
lower=0.01,
upper=0.99,
log=False)
r6_3 = UniformFloatHyperparameter("r6_3",
lower=0.01,
upper=0.99,
log=False)
r6_4 = Constant("r6_4", 0.5)
r7_1 = UniformFloatHyperparameter("r7_1",
lower=0.01,
upper=0.99,
log=False)
r7_2 = UniformFloatHyperparameter("r7_2",
lower=0.01,
upper=0.99,
log=False)
r7_3 = Constant("r7_3", 0.5)
r7_4 = Constant("r7_4", 0.5)
r8_1 = UniformFloatHyperparameter("r8_1",
lower=0.01,
upper=0.99,
log=False)
r8_2 = UniformFloatHyperparameter("r8_2",
lower=0.01,
upper=0.99,
log=False)
r8_3 = Constant("r8_3", 0.5)
r8_4 = Constant("r8_4", 0.5)
r9_1 = UniformFloatHyperparameter("r9_1",
lower=0.01,
upper=0.99,
log=False)
r9_2 = UniformFloatHyperparameter("r9_2",
lower=0.01,
upper=0.99,
log=False)
r9_3 = Constant("r9_3", 0.5)
r9_4 = Constant("r9_4", 0.5)
r10_1 = UniformFloatHyperparameter("r10_1",
lower=0.01,
upper=0.99,
log=False)
r10_2 = UniformFloatHyperparameter("r10_2",
lower=0.01,
upper=0.99,
log=False)
r10_3 = Constant("r10_3", 0.5)
r10_4 = Constant("r10_4", 0.5)
r11_1 = UniformFloatHyperparameter("r11_1",
lower=0.01,
upper=0.99,
log=False)
r11_2 = UniformFloatHyperparameter("r11_2",
lower=0.01,
upper=0.99,
log=False)
r11_3 = Constant("r11_3", 0.5)
r11_4 = Constant("r11_4", 0.5)
r12_1 = UniformFloatHyperparameter("r12_1",
lower=0.01,
upper=0.99,
log=False)
r12_2 = UniformFloatHyperparameter("r12_2",
lower=0.01,
upper=0.99,
log=False)
r12_3 = Constant("r12_3", 0.5)
r12_4 = Constant("r12_4", 0.5)
r13_1 = UniformFloatHyperparameter("r13_1",
lower=0.01,
upper=0.99,
log=False)
r13_2 = UniformFloatHyperparameter("r13_2",
lower=0.01,
upper=0.99,
log=False)
r13_3 = Constant("r13_3", 0.5)
r13_4 = Constant("r13_4", 0.5)
r14_1 = UniformFloatHyperparameter("r14_1",
lower=0.01,
upper=0.99,
log=False)
r14_2 = UniformFloatHyperparameter("r14_2",
lower=0.01,
upper=0.99,
log=False)
r14_3 = Constant("r14_3", 0.5)
r14_4 = Constant("r14_4", 0.5)
cs.add_hyperparameters([
root,
x1,
x2,
x3,
x4,
x5,
x6,
r1_1,
r1_2,
r1_3,
r1_4,
r2_1,
r2_2,
r2_3,
r2_4,
r3_1,
r3_2,
r3_3,
r3_4,
r4_1,
r4_2,
r4_3,
r4_4,
r5_1,
r5_2,
r5_3,
r5_4,
r6_1,
r6_2,
r6_3,
r6_4,
r7_1,
r7_2,
r7_3,
r7_4,
r8_1,
r8_2,
r8_3,
r8_4,
r9_1,
r9_2,
r9_3,
r9_4,
r10_1,
r10_2,
r10_3,
r10_4,
r11_1,
r11_2,
r11_3,
r11_4,
r12_1,
r12_2,
r12_3,
r12_4,
r13_1,
r13_2,
r13_3,
r13_4,
r14_1,
r14_2,
r14_3,
r14_4,
])
# add condition
cs.add_condition(InCondition(x1, root, ["l1"]))
cs.add_condition(InCondition(x2, root, ["ln"]))
cs.add_condition(InCondition(r1_1, root, ["l1"]))
cs.add_condition(InCondition(r1_2, root, ["l1"]))
cs.add_condition(InCondition(r1_3, root, ["l1"]))
cs.add_condition(InCondition(r1_4, root, ["l1"]))
cs.add_condition(InCondition(r2_1, root, ["ln"]))
cs.add_condition(InCondition(r2_2, root, ["ln"]))
cs.add_condition(InCondition(r2_3, root, ["ln"]))
cs.add_condition(InCondition(r2_4, root, ["ln"]))
cs.add_condition(InCondition(x3, x1, ["l1"]))
cs.add_condition(InCondition(x4, x1, ["ln"]))
cs.add_condition(InCondition(r3_1, x1, ["l1"]))
cs.add_condition(InCondition(r3_2, x1, ["l1"]))
cs.add_condition(InCondition(r3_3, x1, ["l1"]))
cs.add_condition(InCondition(r3_4, x1, ["l1"]))
cs.add_condition(InCondition(r4_1, x1, ["ln"]))
cs.add_condition(InCondition(r4_2, x1, ["ln"]))
cs.add_condition(InCondition(r4_3, x1, ["ln"]))
cs.add_condition(InCondition(r4_4, x1, ["ln"]))
cs.add_condition(InCondition(x5, x2, ["l1"]))
cs.add_condition(InCondition(x6, x2, ["ln"]))
cs.add_condition(InCondition(r5_1, x2, ["l1"]))
cs.add_condition(InCondition(r5_2, x2, ["l1"]))
cs.add_condition(InCondition(r5_3, x2, ["l1"]))
cs.add_condition(InCondition(r5_4, x2, ["l1"]))
cs.add_condition(InCondition(r6_1, x2, ["ln"]))
cs.add_condition(InCondition(r6_2, x2, ["ln"]))
cs.add_condition(InCondition(r6_3, x2, ["ln"]))
cs.add_condition(InCondition(r6_4, x2, ["ln"]))
cs.add_condition(InCondition(r7_1, x3, ["l1"]))
cs.add_condition(InCondition(r7_2, x3, ["l1"]))
cs.add_condition(InCondition(r7_3, x3, ["l1"]))
cs.add_condition(InCondition(r7_4, x3, ["l1"]))
cs.add_condition(InCondition(r8_1, x3, ["ln"]))
cs.add_condition(InCondition(r8_2, x3, ["ln"]))
cs.add_condition(InCondition(r8_3, x3, ["ln"]))
cs.add_condition(InCondition(r8_4, x3, ["ln"]))
cs.add_condition(InCondition(r9_1, x4, ["l1"]))
cs.add_condition(InCondition(r9_2, x4, ["l1"]))
cs.add_condition(InCondition(r9_3, x4, ["l1"]))
cs.add_condition(InCondition(r9_4, x4, ["l1"]))
cs.add_condition(InCondition(r10_1, x4, ["ln"]))
cs.add_condition(InCondition(r10_2, x4, ["ln"]))
cs.add_condition(InCondition(r10_3, x4, ["ln"]))
cs.add_condition(InCondition(r10_4, x4, ["ln"]))
cs.add_condition(InCondition(r11_1, x5, ["l1"]))
cs.add_condition(InCondition(r11_2, x5, ["l1"]))
cs.add_condition(InCondition(r11_3, x5, ["l1"]))
cs.add_condition(InCondition(r11_4, x5, ["l1"]))
cs.add_condition(InCondition(r12_1, x5, ["ln"]))
cs.add_condition(InCondition(r12_2, x5, ["ln"]))
cs.add_condition(InCondition(r12_3, x5, ["ln"]))
cs.add_condition(InCondition(r12_4, x5, ["ln"]))
cs.add_condition(InCondition(r13_1, x6, ["l1"]))
cs.add_condition(InCondition(r13_2, x6, ["l1"]))
cs.add_condition(InCondition(r13_3, x6, ["l1"]))
cs.add_condition(InCondition(r13_4, x6, ["l1"]))
cs.add_condition(InCondition(r14_1, x6, ["ln"]))
cs.add_condition(InCondition(r14_2, x6, ["ln"]))
cs.add_condition(InCondition(r14_3, x6, ["ln"]))
cs.add_condition(InCondition(r14_4, x6, ["ln"]))
return cs
class ClusteringArmThread:
def __init__(self, name, metric, X):
self.thread_name = name
self.metric = metric
self.X = X
self.clu_cs = ConfigurationSpace()
if (name == Constants.kmeans_algo):
algorithm = CategoricalHyperparameter("algorithm", ["auto", "full", "elkan"])
tol = UniformFloatHyperparameter("tol", 1e-6, 1e-2)
n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 15)
n_init = UniformIntegerHyperparameter("n_init", 2, 15)
max_iter = UniformIntegerHyperparameter("max_iter", 50, 1500)
verbose = Constant("verbose", 0)
self.clu_cs.add_hyperparameters([n_clusters, n_init, max_iter, tol, verbose, algorithm])
elif (name == Constants.affinity_algo):
damping = UniformFloatHyperparameter("damping", 0.5, 1.0)
max_iter = UniformIntegerHyperparameter("max_iter", 100, 1000)
convergence_iter = UniformIntegerHyperparameter("convergence_iter", 5, 20)
self.clu_cs.add_hyperparameters([damping, max_iter, convergence_iter])
elif (name == Constants.mean_shift_algo):
quantile = UniformFloatHyperparameter("quantile", 0.0, 1.0)
bin_seeding = UniformIntegerHyperparameter("bin_seeding", 0, 1)
min_bin_freq = UniformIntegerHyperparameter("min_bin_freq", 1, 100)
cluster_all = UniformIntegerHyperparameter("cluster_all", 0, 1)
self.clu_cs.add_hyperparameters([quantile, bin_seeding, min_bin_freq, cluster_all])
elif (name == Constants.ward_algo):
linkage = CategoricalHyperparameter("linkage", ["ward", "complete", "average"])
affinity_all = CategoricalHyperparameter("affinity_a",
["euclidean", "l1", "l2", "manhattan", "cosine", "precomputed"])
affinity_ward = CategoricalHyperparameter("affinity_w", ["euclidean"])
n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 15)
self.clu_cs.add_hyperparameters([n_clusters, affinity_all, affinity_ward, linkage])
self.clu_cs.add_condition(InCondition(child=affinity_ward, parent=linkage, values=["ward"]))
self.clu_cs.add_condition(
InCondition(child=affinity_all, parent=linkage, values=["ward", "complete", "average"]))
elif (name == Constants.dbscan_algo):
algorithm = CategoricalHyperparameter("algorithm", ["auto", "ball_tree", "kd_tree", "brute"])
eps = UniformFloatHyperparameter("eps", 0.1, 0.9)
min_samples = UniformIntegerHyperparameter("min_samples", 2, 10)
leaf_size = UniformIntegerHyperparameter("leaf_size", 5, 100)
self.clu_cs.add_hyperparameters([eps, min_samples, algorithm, leaf_size])
elif (name == Constants.gm_algo):
cov_t = CategoricalHyperparameter("covariance_type", ["full", "tied", "diag", "spherical"])
tol = UniformFloatHyperparameter("tol", 1e-6, 0.1)
reg_c = UniformFloatHyperparameter("reg_covar", 1e-10, 0.1)
n_com = UniformIntegerHyperparameter("n_components", 2, 15)
max_iter = UniformIntegerHyperparameter("max_iter", 10, 1000)
self.clu_cs.add_hyperparameters([cov_t, tol, reg_c, n_com, max_iter])
elif (name == Constants.bgm_algo):
cov_t = CategoricalHyperparameter("covariance_type", ["full", "tied", "diag", "spherical"])
tol = UniformFloatHyperparameter("tol", 1e-6, 0.1)
reg_c = UniformFloatHyperparameter("reg_covar", 1e-10, 0.1)
wcp = UniformFloatHyperparameter("weight_concentration_prior", 1e-10, 0.1)
mpp = UniformFloatHyperparameter("mean_precision_prior", 1e-10, 0.1)
n_com = UniformIntegerHyperparameter("n_components", 2, 15)
max_iter = UniformIntegerHyperparameter("max_iter", 10, 1000)
self.clu_cs.add_hyperparameters([wcp, mpp, cov_t, tol, reg_c, n_com, max_iter])
def clu_run(self, cfg):
cl = None
if (self.thread_name == Constants.kmeans_algo):
cl = KMeans(**cfg)
elif (self.thread_name == Constants.affinity_algo):
cl = AffinityPropagation(**cfg)
elif (self.thread_name == Constants.mean_shift_algo):
bandwidth = estimate_bandwidth(self.X, quantile=cfg['quantile'])
cl = MeanShift(bandwidth=bandwidth, bin_seeding=bool(cfg['bin_seeding']), min_bin_freq=cfg['min_bin_freq'],
cluster_all=bool(cfg['cluster_all']))
elif (self.thread_name == Constants.ward_algo):
linkage = cfg["linkage"]
aff = ""
if ("ward" in linkage):
aff = cfg["affinity_w"]
else:
aff = cfg["affinity_a"]
n_c = cfg["n_clusters"]
cl = AgglomerativeClustering(n_clusters=n_c, linkage=linkage, affinity=aff)
elif (self.thread_name == Constants.dbscan_algo):
cl = DBSCAN(**cfg)
elif (self.thread_name == Constants.gm_algo):
cl = GaussianMixture(**cfg)
elif (self.thread_name == Constants.bgm_algo):
cl = BayesianGaussianMixture(**cfg)
cl.fit(self.X)
if (self.thread_name == Constants.gm_algo) or (self.thread_name == Constants.bgm_algo):
labels = cl.predict(self.X)
else:
labels = cl.labels_
# labels_unique = np.unique(labels)
# n_clusters = len(labels_unique)
value = self.metric(self.X, labels)
# print(self.thread_name + " n_clusters=" + str(n_clusters) + " metric=" + str(value))
return value
def main_loop(problem):
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators",
5,
50,
default_value=10)
#criterion = CategoricalHyperparameter("criterion", ["mse", "mae"], default_value="mse")
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
1,
20,
default_value=1)
min_weight_fraction_leaf = UniformFloatHyperparameter(
"min_weight_fraction_leaf", 0.0, 0.5, default_value=0.0)
max_leaf_nodes = UniformIntegerHyperparameter("max_leaf_nodes",
10,
1000,
default_value=100)
min_impurity_decrease = UniformFloatHyperparameter("min_impurity_decrease",
0.0,
0.5,
default_value=0.0)
warm_start = CategoricalHyperparameter("warm_start", ["true", "false"],
default_value="false")
cs.add_hyperparameters([
n_estimators, min_weight_fraction_leaf, min_samples_split,
min_samples_leaf, max_leaf_nodes, warm_start, min_impurity_decrease
])
max_features = CategoricalHyperparameter(
"max_features", ["auto", "log2", "sqrt", "int", "None", "float"],
default_value="auto") # only rbf, poly, sigmoid
max_features_int = UniformIntegerHyperparameter("max_features_int",
2,
len(X[0]),
default_value=5)
max_features_float = UniformFloatHyperparameter("max_features_float",
0.0,
0.9,
default_value=0.0)
cs.add_hyperparameters(
[max_features, max_features_int, max_features_float])
use_max_features_int = InCondition(child=max_features_int,
parent=max_features,
values=["int"])
use_max_features_float = InCondition(child=max_features_float,
parent=max_features,
values=["float"])
cs.add_conditions([use_max_features_int, use_max_features_float])
max_depth = CategoricalHyperparameter("max_depth", ["None", "value"],
default_value="None")
max_depth_value = UniformIntegerHyperparameter("max_depth_value",
2,
20,
default_value=5)
cs.add_hyperparameters([max_depth, max_depth_value])
cs.add_condition(
InCondition(child=max_depth_value, parent=max_depth, values=["value"]))
random_state = CategoricalHyperparameter("random_state", ["None", "value"],
default_value="None")
random_state_value = UniformIntegerHyperparameter("random_state_value",
1,
20,
default_value=1)
cs.add_hyperparameters([random_state, random_state_value])
cs.add_condition(
InCondition(child=random_state_value,
parent=random_state,
values=["value"]))
with open("/home/naamah/Documents/CatES/result_All/X1.p", "rb") as fp:
X = pickle.load(fp)
# Scenario object
max_eval = 100000
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit":
max_eval, # maximum function evaluations
"cs":
cs, # configuration space
"shared_model":
True,
"output_dir":
"/home/naamah/Documents/CatES/result_All/smac/RF/run_{}_{}_{}".format(
max_eval,
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem),
"input_psmac_dirs":
"/home/naamah/Documents/CatES/result_All/smac/psmac",
"deterministic":
"False"
})
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
# main_loop()
class SMACOptimizer(PhotonMasterOptimizer):
def __init__(self, facade='SMAC4HPO', scenario_dict=None, intensifier_kwargs=None, rng=42, smac_helper=None):
"""
SMAC Wrapper for PHOTON.
SMAC usage and implementation details:
https://automl.github.io/SMAC3/master/quickstart.html
Parameters
----------
* `facade` [str or smac.facade.class, default: 'SMAC4HPO']:
Choice of SMAC backend strategy, [SMAC4BO, SMAC4HPO, SMAC4AC].
* `scenario_dict` [dict, default: None (warning scenario with wallclock_limit = 60*40)]
Informations for scenario settings like run_limit or wallclock_limit.
Different to main SMAC cs (configspace) is not required or used cause PHOTON translate own param_space
to SMAC.configspace.
* `rng`: [int, default: 42]
random seed of SMAC.facade
* `smac_helper` [dict]
For testing this object give public access to SMAC.facade object.
Currently help object for test cases.
"""
super(SMACOptimizer, self).__init__()
if not __found__:
msg = "Module smac not found or not installed as expected. " \
"Please install the smac_requirements.txt PHOTON provides."
logger.error(msg)
raise ModuleNotFoundError(msg)
if not scenario_dict:
self.scenario_dict = {"run_obj": "quality",
"deterministic": "true",
"wallclock_limit": 60 * 40}
msg = "No scenario_dict for smac was given. Falling back to default values: {}.".format(self.scenario_dict)
logger.warning(msg)
else:
self.scenario_dict = scenario_dict
if facade in ["SMAC4BO", SMAC4BO, "SMAC4AC", SMAC4AC, "SMAC4HPO", SMAC4HPO, "BOHB4HPO", BOHB4HPO]:
if type(facade) == str:
self.facade = eval(facade)
else:
self.facade = facade
else:
msg = "SMAC.facade {} not known. Please use one of ['SMAC4BO', 'SMAC4AC', 'SMAC4HPO']."
logger.error(msg.format(str(facade)))
raise ValueError(msg.format(str(facade)))
self.rng = rng
if not intensifier_kwargs:
self.intensifier_kwargs = {}
else:
self.intensifier_kwargs = intensifier_kwargs
self.cspace = ConfigurationSpace() # Hyperparameter space for SMAC
self.switch_optiones = {}
self.hyperparameters = []
if smac_helper:
self.smac_helper = smac_helper
self.debug = True
else:
self.debug = False
self.maximize_metric = False
@staticmethod
def _convert_photon_to_smac_param(hyperparam: object, name: str):
"""
Helper function: Convert PHOTON hyperparameter to SMAC hyperparameter.
Parameters
----------
* `hyperparam` [object]:
One of photonai.optimization.hyperparameters.
* `name` [str]
Name of hyperparameter.
"""
if not hyperparam:
return None
if isinstance(hyperparam, PhotonCategorical):
return CategoricalHyperparameter(name, hyperparam.values)
elif isinstance(hyperparam, list):
return CategoricalHyperparameter(name, hyperparam)
elif isinstance(hyperparam, FloatRange):
if hyperparam.range_type == 'linspace':
return UniformFloatHyperparameter(name, hyperparam.start, hyperparam.stop)
elif hyperparam.range_type == 'logspace':
raise NotImplementedError("Logspace in your float hyperparameter is not implemented in SMAC.")
else:
return UniformFloatHyperparameter(name, hyperparam.start, hyperparam.stop)
elif isinstance(hyperparam, IntegerRange):
return UniformIntegerHyperparameter(name, hyperparam.start, hyperparam.stop)
def build_smac_space(self, pipeline_elements):
"""
Build entire SMAC hyperparameter space.
Parameters
----------
* `pipeline_elements` [list]:
List of all pipeline_elements to create hyperparameter_space.
"""
for pipe_element in pipeline_elements:
# build conditions for switch elements
if pipe_element.__class__.__name__ == 'Switch':
algorithm_options = {}
for algo in pipe_element.elements:
algo_params = [] # hyper params corresponding to "algo"
for name, value in algo.hyperparameters.items():
smac_param = self._convert_photon_to_smac_param(value, (
pipe_element.name + "__" + name)) # or element.name__algo.name__name
algo_params.append(smac_param)
algorithm_options[(pipe_element.name + "__" + algo.name)] = algo_params
algos = CategoricalHyperparameter(pipe_element.name + "__algos", choices=algorithm_options.keys())
self.switch_optiones[pipe_element.name + "__algos"] = algorithm_options.keys()
self.cspace.add_hyperparameter(algos)
for algo, params in algorithm_options.items():
for param in params:
cond = InCondition(child=param, parent=algos, values=[algo])
self.cspace.add_hyperparameter(param)
self.cspace.add_condition(cond)
continue
if hasattr(pipe_element, 'hyperparameters'):
for name, value in pipe_element.hyperparameters.items():
self.hyperparameters.append(name)
# if we only have one value we do not need to optimize
if isinstance(value, list) and len(value) < 2:
self.constant_dictionary[name] = value[0]
continue
if isinstance(value, PhotonCategorical) and len(value.values) < 2:
self.constant_dictionary[name] = value.values[0]
continue
smac_param = self._convert_photon_to_smac_param(value, name)
if smac_param is not None:
self.cspace.add_hyperparameter(smac_param)
def prepare(self, pipeline_elements: list, maximize_metric: bool, objective_function):
"""
Initializes SMAC Optimizer.
Parameters
----------
* `pipeline_elements` [list]:
List of all pipeline_elements to create hyperparameter_space.
* `maximize_metric` [bool]:
Boolean for distinguish between score and error.
* `objective_function` [callable]:
The cost or objective function.
"""
self.space = ConfigurationSpace() # build space
self.build_smac_space(pipeline_elements)
self.maximize_metric = maximize_metric
self.scenario_dict["cs"] = self.cspace
self.scenario_dict["limit_resources"] = False
self.scenario = Scenario(self.scenario_dict)
def smac_objective_function(current_config):
current_config = {k: current_config[k] for k in current_config if (current_config[k] and 'algos' not in k)}
return objective_function(current_config)
self.smac = self.facade(scenario=self.scenario,
intensifier_kwargs=self.intensifier_kwargs,
rng=self.rng,
tae_runner=smac_objective_function)
if self.debug:
self.smac_helper['data'] = self.smac
def optimize(self):
"""
Start optimization process.
"""
self.smac.optimize()
def main_loop(problem):
#logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
cs = ConfigurationSpace()
kernel = CategoricalHyperparameter("kernel", ["rbf", "sigmoid"],
default_value="rbf")
cs.add_hyperparameter(kernel)
C = UniformFloatHyperparameter("C", 1, 1000.0, default_value=900.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"],
default_value="false")
cs.add_hyperparameters([C, shrinking])
#degree = UniformIntegerHyperparameter("degree", 1, 3 ,default_value=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 1.0,
default_value=0.0) # poly, sigmoid
#cs.add_hyperparameters([degree, coef0])
cs.add_hyperparameter(coef0)
#use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0, parent=kernel, values=["sigmoid"])
#cs.add_conditions([use_degree, use_coef0])
cs.add_condition(use_coef0)
gamma = CategoricalHyperparameter(
"gamma", ["auto", "value"],
default_value="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value",
0.001,
8,
default_value=1)
cs.add_hyperparameters([gamma, gamma_value])
cs.add_condition(
InCondition(child=gamma_value, parent=gamma, values=["value"]))
cs.add_condition(
InCondition(child=gamma, parent=kernel, values=["rbf", "sigmoid"]))
epsilon = UniformFloatHyperparameter("epsilon",
0.001,
5.0,
default_value=0.1)
cs.add_hyperparameter(epsilon)
# Scenario object
max_eval = 100000
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit":
max_eval, # maximum function evaluations
"cs":
cs, # configuration space
"shared_model":
True,
"output_dir":
"/home/naamah/Documents/CatES/result_All/smac/svm/run_{}_{}_{}".format(
max_eval,
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d_%H:%M:%S'), problem),
# "input_psmac_dirs": "/home/naamah/Documents/CatES/result_All/smac/svm/{}/run_{}_{}/*".format(problem,max_eval, datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S_%f')),
"deterministic":
"False"
#"instance_file":"/home/naamah/Documents/CatES/result_All",
#"test_instance_file":"/home/naamah/Documents/CatES/result_All"
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
def_value = svm_from_cfg(cs.get_default_configuration())
print("Default Value: %.2f" % (def_value))
# Optimize, using a SMAC-object
print(
"Optimizing! Depending on your machine, this might take a few minutes."
)
smac = SMAC(scenario=scenario, tae_runner=svm_from_cfg)
incumbent = smac.optimize()
inc_value = svm_from_cfg(incumbent)
print("Optimized Value: %.2f" % (inc_value))
return (incumbent)
def generate_data(smac_class,
n_runs=1,
output_dir: Union[str, Path] = ".",
dataset=None,
runcount_limit=50):
output_dir = Path(output_dir)
if dataset is None:
dataset = datasets.load_iris()
def svm_from_cfg(cfg):
""" Creates a SVM based on a configuration and evaluates it on the
iris-dataset using cross-validation.
Parameters:
-----------
cfg: Configuration (ConfigSpace.ConfigurationSpace.Configuration)
Configuration containing the parameters.
Configurations are indexable!
Returns:
--------
A crossvalidated mean score for the svm on the loaded data2-set.
"""
# For deactivated parameters, the configuration stores None-values.
# This is not accepted by the SVM, so we remove them.
cfg = {k: cfg[k] for k in cfg if cfg[k]}
# We translate boolean values:
cfg["shrinking"] = True if cfg["shrinking"] == "true" else False
# And for gamma, we set it to a fixed value or to "auto" (if used)
if "gamma" in cfg:
cfg["gamma"] = cfg["gamma_value"] if cfg[
"gamma"] == "value" else "auto"
cfg.pop("gamma_value", None) # Remove "gamma_value"
clf = svm.SVC(**cfg, random_state=None)
scores = cross_val_score(clf, dataset.data, dataset.target, cv=5)
return 1 - np.mean(scores) # Minimize!
# logger = logging.getLogger("SVMExample")
logging.basicConfig(level=logging.INFO) # logging.DEBUG for debug output
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
# We define a few possible types of SVM-kernels and add them as "kernel" to our cs
kernel = CategoricalHyperparameter("kernel",
["linear", "rbf", "poly", "sigmoid"],
default_value="poly")
cs.add_hyperparameter(kernel)
# There are some hyperparameters shared by all kernels
C = UniformFloatHyperparameter("C", 0.001, 1000.0, default_value=1.0)
shrinking = CategoricalHyperparameter("shrinking", ["true", "false"],
default_value="true")
cs.add_hyperparameters([C, shrinking])
# Others are kernel-specific, so we can add conditions to limit the searchspace
degree = UniformIntegerHyperparameter(
"degree", 1, 5, default_value=3) # Only used by kernel poly
coef0 = UniformFloatHyperparameter("coef0", 0.0, 10.0,
default_value=0.0) # poly, sigmoid
cs.add_hyperparameters([degree, coef0])
use_degree = InCondition(child=degree, parent=kernel, values=["poly"])
use_coef0 = InCondition(child=coef0,
parent=kernel,
values=["poly", "sigmoid"])
cs.add_conditions([use_degree, use_coef0])
# This also works for parameters that are a mix of categorical and values from a range of numbers
# For example, gamma can be either "auto" or a fixed float
gamma = CategoricalHyperparameter(
"gamma", ["auto", "value"],
default_value="auto") # only rbf, poly, sigmoid
gamma_value = UniformFloatHyperparameter("gamma_value",
0.0001,
8,
default_value=1)
cs.add_hyperparameters([gamma, gamma_value])
# We only activate gamma_value if gamma is set to "value"
cs.add_condition(
InCondition(child=gamma_value, parent=gamma, values=["value"]))
# And again we can restrict the use of gamma in general to the choice of the kernel
cs.add_condition(
InCondition(child=gamma,
parent=kernel,
values=["rbf", "poly", "sigmoid"]))
# Scenario object
for i in range(n_runs):
scenario = Scenario({
"run_obj":
"quality", # we optimize quality (alternatively runtime)
"runcount-limit":
runcount_limit,
# max. number of function evaluations; for this example set to a low number
"cs":
cs, # configuration space
"deterministic":
"true",
"limit_resources":
"false",
"output_dir":
str((output_dir / smac_class.__name__ / f"{i:02d}").absolute())
})
# Example call of the function
# It returns: Status, Cost, Runtime, Additional Infos
# def_value = svm_from_cfg(cs.get_default_configuration())
# print(f"Default Value: {def_value:.2f}")
#
# Optimize, using a SMAC-object
smac = smac_class(scenario=scenario, rng=None, tae_runner=svm_from_cfg)
incumbent = smac.optimize()
#
inc_value = svm_from_cfg(incumbent)
#
# print(f"Optimized Value: {inc_value:.2f}")
#
# # We can also validate our results (though this makes a lot more sense with instances)
smac.validate(
config_mode='inc', # We can choose which configurations to evaluate
# instance_mode='train+test', # Defines what instances to validate
repetitions=
100, # Ignored, unless you set "deterministic" to "false" in line 95
n_jobs=1) # How many cores to use in parallel for optimization
logger = logging.getLogger("Optimizer") # Enable to show Debug outputs
logging.basicConfig(level=logging.DEBUG)
# build Configuration Space which defines all parameters and their ranges
n_params = 16
use_conditionals = True # using conditionals should help a lot in this example
cs = ConfigurationSpace()
previous_param = None
for n in range(n_params):
p = CategoricalHyperparameter("%d" % (n), [0, 1], default=0)
cs.add_hyperparameter(p)
if n > 0 and use_conditionals:
cond = InCondition(child=p, parent=previous_param, values=[1])
cs.add_condition(cond)
previous_param = p
# SMAC scenario object
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternative runtime)
"runcount-limit": n_params * 2, # at most 200 function evaluations
"cs": cs, # configuration space
"deterministic": "true"
})
# register function to be optimize
taf = ExecuteTAFuncDict(leading_ones)
# example call of the function
