def set_probabilities_in_cs(self, cs: ConfigurationSpace,
relied2models: Dict[str, List[str]],
relied2AllModels: Dict[str, List[str]],
all_models: List[str], **kwargs):
estimator = cs.get_hyperparameter(f"{PHASE2}:__choice__")
probabilities = []
model2prob = {}
L = 0
for rely_model in relied2models:
cur_models = relied2models[rely_model]
L += len(cur_models)
for model in cur_models:
model2prob[model] = kwargs[rely_model] / len(cur_models)
p_rest = (1 - sum(model2prob.values())) / (len(all_models) - L)
for model in estimator.choices:
probabilities.append(model2prob.get(model, p_rest))
estimator.probabilities = probabilities
default_estimator_choice = None
for models in relied2models.values():
if models:
default_estimator_choice = models[0]
estimator.default_value = default_estimator_choice
for rely_model, path in RelyModels.info:
forbid_eq_value = path[-1]
path = path[:-1]
forbid_eq_key = ":".join(path + ["__choice__"])
forbid_eq_key_hp = cs.get_hyperparameter(forbid_eq_key)
forbid_in_key = f"{PHASE2}:__choice__"
hit = relied2AllModels.get(rely_model)
if not hit:
choices = list(forbid_eq_key_hp.choices)
choices.remove(forbid_eq_value)
forbid_eq_key_hp.choices = tuple(choices)
forbid_eq_key_hp.default_value = choices[0]
forbid_eq_key_hp.probabilities = [1 / len(choices)
] * len(choices)
# fixme 最后我放弃了在这上面进行修改，在hdl部分就做了预处理
continue
forbid_in_value = list(set(all_models) - set(hit))
# 只选择了boost模型
if not forbid_in_value:
continue
choices = forbid_eq_key_hp.choices
probabilities = []
p: float = kwargs[rely_model]
p_rest = (1 - p) * (len(choices) - 1)
for choice in choices:
if choice == forbid_eq_value:
probabilities.append(p)
else:
probabilities.append(p_rest)
forbid_eq_key_hp.probabilities = probabilities
cs.add_forbidden_clause(
ForbiddenAndConjunction(
ForbiddenEqualsClause(forbid_eq_key_hp, forbid_eq_value),
ForbiddenInClause(cs.get_hyperparameter(forbid_in_key),
forbid_in_value),
))
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def get_random_initial_configs(shps: ConfigurationSpace,
n_configs,
random_state=42) -> List[Configuration]:
None_name = "None:NoneType"
shps = deepcopy(shps)
shps.seed(random_state)
for config in shps.get_hyperparameters():
name: str = config.name
if name.startswith(PHASE1) and name.endswith("__choice__") and (
None_name in config.choices): # fixme 重构之后 None_name是不是改变了？
config.default_value = None_name
model_choice = shps.get_hyperparameter(f"{PHASE2}:__choice__")
result = []
for choice in model_choice.choices:
cur_phps = deepcopy(shps)
cur_phps.get_hyperparameter(
f"{PHASE2}:__choice__").default_value = choice
default = cur_phps.get_default_configuration()
result.append(default)
if len(result) < n_configs:
result.extend(shps.sample_configuration(n_configs - len(result)))
elif len(result) > n_configs:
result = random.sample(result, n_configs)
return result
def map_to_config(cs: CS.ConfigurationSpace,
vector: Union[np.array, List]) -> CS.Configuration:
"""Return a ConfigSpace object with values assigned from the vector
Parameters
----------
cs : ConfigSpace.ConfigurationSpace
vector : np.array or list
Returns
-------
ConfigSpace.Configuration
"""
config = cs.sample_configuration()
for i, name in enumerate(np.sort(cs.get_hyperparameter_names())):
hp = cs.get_hyperparameter(str(name))
if isinstance(hp, CS.UniformIntegerHyperparameter):
config[hp.name] = int(vector[i])
elif isinstance(hp, CS.UniformFloatHyperparameter):
# clip introduced to handle the edge case when a 64-bit float type casting introduces
# extra precision which can make the number lower than the hard hp.lower limit
config[hp.name] = np.clip(float(vector[i]), hp.lower, hp.upper)
else:
config[hp.name] = vector[i]
return config
def revert_eval_config(eval_config: Dict[str, NumericType],
config_space: CS.ConfigurationSpace,
is_categoricals: Dict[str,
bool], is_ordinals: Dict[str,
bool],
hp_names: List[str]) -> Dict[str, Any]:
"""
Revert the eval_config into the original value range.
For example,
* categorical: index of a symbol -> the symbol
* log float: log(value) -> exp(log(value)) = value
* quantized value: value -> steped by q, i.e. q, 2q, 3q, ...
Args:
eval_config (Dict[str, NumericType]): The configuration to evaluate and revert
config_space (CS.ConfigurationSpace): The searching space information
is_categoricals (Dict[str, bool]): Whether each hyperparameter is categorical
is_ordinals (Dict[str, bool]): Whether each hyperparameter is ordinal
hp_names (List[str]): The list of the names of hyperparameters
Returns:
reverted_eval_config (Dict[str, Any])
"""
converted_eval_config: Dict[str, Any] = {}
for hp_name in hp_names:
is_categorical, is_ordinal = is_categoricals[hp_name], is_ordinals[
hp_name]
config = config_space.get_hyperparameter(hp_name)
val = eval_config[hp_name]
if is_categorical:
converted_eval_config[hp_name] = config.choices[val]
elif is_ordinal:
if config.meta is None:
raise ValueError(
'The meta information of the ordinal hyperparameter must be provided'
)
log = config.meta.get('log', False)
vals = np.log(config.sequence) if log else np.array(
config.sequence)
diff = np.abs(vals - val)
converted_eval_config[hp_name] = config.sequence[diff.argmin()]
else:
dtype = config2type[config.__class__.__name__]
q = config.q
if config.log:
val = np.exp(val)
if q is not None or dtype is int:
q = 1 if q is None and dtype is int else q
val = np.round(val / q) * q
check_value_range(hp_name=hp_name, config=config, val=val)
converted_eval_config[hp_name] = dtype(val)
return converted_eval_config
def get_fidelity_grid(
cs: CS.ConfigurationSpace,
grid_step_size: int = 10,
convert_to_configspace: bool = False,
include_sh_budgets: bool = True
) -> Union[List[Tuple], List[CS.Configuration]]:
"""Generates a grid from cartesian product of the fidelity spaced out at given step size
Parameters
----------
cs : ConfigSpace.ConfigurationSpace
grid_step_size : int
The number of steps to divide a parameter dimension into
convert_to_configspace : bool
If True, returns a list of ConfigSpace objects of each point in the grid
If False, returns a list of tuples containing the values of each point in grid
include_sh_budgets : bool
If True, additionally includes budget spacing from Hyperband for eta={2,3,4}
Returns
-------
list
"""
param_ranges = []
selected_etas = [3]
for name in np.sort(cs.get_hyperparameter_names()):
hp = cs.get_hyperparameter(str(name))
if isinstance(hp, CS.Constant):
param_ranges.append([hp.value])
else:
if hp.log:
grid = np.exp(
np.linspace(np.log(hp.lower), np.log(hp.upper),
grid_step_size))
grid = np.clip(grid, hp.lower, hp.upper) #.astype(np.float32)
else:
grid = np.linspace(hp.lower, hp.upper,
grid_step_size) #, dtype=np.float32)
if include_sh_budgets:
hb_grid = np.array([])
for eta in selected_etas:
hb_grid = np.concatenate(
(hb_grid,
generate_SH_fidelities(hp.lower, hp.upper,
eta))) # .astype(np.float32)
grid = np.unique(np.concatenate((hb_grid, grid)))
grid = grid.astype(int) if isinstance(
hp, CS.UniformIntegerHyperparameter) else grid
param_ranges.append(np.unique(grid))
full_grid = itertools.product(*param_ranges)
if not convert_to_configspace:
return list(full_grid)
config_list = []
for _config in full_grid:
config_list.append(map_to_config(cs, _config))
return config_list
def cast_columns_of_dataframe(df: pd.DataFrame, params: List,
config_space: ConfigSpace.ConfigurationSpace):
for param in params:
hyperparameter = config_space.get_hyperparameter(param)
if isinstance(hyperparameter, ConfigSpace.UniformIntegerHyperparameter) or \
(isinstance(hyperparameter, ConfigSpace.Constant) and isinstance(hyperparameter.value, int)) or \
(isinstance(hyperparameter, ConfigSpace.UnParametrizedHyperparameter) and isinstance(hyperparameter.value, int)):
# limitation of pandas: can't mix nan and integer
df[param] = df[param].dropna().apply(lambda x: str(int(x)))
return df
def get_parameter_grid(
cs: CS.ConfigurationSpace,
grid_step_size: int = 10,
convert_to_configspace: bool = False
) -> Union[List[Tuple], List[CS.Configuration]]:
"""Generates a grid from cartesian product of the parameters spaced out at given step size
Parameters
----------
cs : ConfigSpace.ConfigurationSpace
grid_step_size : int
The number of steps to divide a parameter dimension into
convert_to_configspace : bool
If True, returns a list of ConfigSpace objects of each point in the grid
If False, returns a list of tuples containing the values of each point in grid
Returns
-------
list
"""
param_ranges = []
for name in np.sort(cs.get_hyperparameter_names()):
hp = cs.get_hyperparameter(str(name))
if isinstance(hp, CS.CategoricalHyperparameter):
param_ranges.append(hp.choices)
elif isinstance(hp, CS.OrdinalHyperparameter):
param_ranges.append(hp.sequence)
elif isinstance(hp, CS.Constant):
param_ranges.append([hp.value])
else:
if hp.log:
grid = np.exp(
np.linspace(np.log(hp.lower), np.log(hp.upper),
grid_step_size))
grid = np.clip(grid, hp.lower, hp.upper).astype(np.float32)
else:
grid = np.linspace(hp.lower,
hp.upper,
grid_step_size,
dtype=np.float32)
grid = grid.astype(int) if isinstance(
hp, CS.UniformIntegerHyperparameter) else grid
grid = np.unique(grid).tolist()
param_ranges.append(grid)
full_grid = itertools.product(*param_ranges)
if not convert_to_configspace:
return list(full_grid)
config_list = []
for _config in full_grid:
config_list.append(map_to_config(cs, _config))
return config_list
def get_discrete_configspace(configspace: CS.ConfigurationSpace,
grid_size: int = 10,
seed: Union[int, None] = None,
fidelity_space: bool = False):
""" Generates a new discretized ConfigurationSpace from a generally defined space
Given the discretization grid size for each dimension, the new ConfigurationSpace contains
each hyperparmater as an OrdinalParameter with the discretized values for that dimension as
the sequence of choices available for that hyperparameter.
Parameters
----------
configspace : ConfigSpace.ConfigurationSpace
grid_size : int
The number of steps to divide a parameter dimension into
seed : int
Returns
-------
ConfigSpace.ConfigurationSpace
"""
if fidelity_space:
grid_list = pd.DataFrame(get_fidelity_grid(configspace, grid_size))
else:
grid_list = pd.DataFrame(get_parameter_grid(configspace, grid_size))
cs = CS.ConfigurationSpace(seed=seed)
hp_names = np.sort(configspace.get_hyperparameter_names()).tolist()
for i, k in enumerate(hp_names):
choices = grid_list.iloc[:, i].unique()
if isinstance(configspace.get_hyperparameter(k),
CS.UniformIntegerHyperparameter):
choices = choices.astype(int)
elif isinstance(configspace.get_hyperparameter(k),
CS.UniformFloatHyperparameter):
choices = choices.astype(np.float32)
cs.add_hyperparameter(CS.OrdinalHyperparameter(str(k), choices))
return cs
def test_meta_field(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp",
lower=1,
upper=10,
meta=dict(uihp=True)))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp",
mu=0,
sigma=1,
meta=dict(nihp=True)))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp",
lower=1,
upper=10,
meta=dict(ufhp=True)))
cs.add_hyperparameter(
NormalFloatHyperparameter("nfhp",
mu=0,
sigma=1,
meta=dict(nfhp=True)))
cs.add_hyperparameter(
CategoricalHyperparameter("chp",
choices=['1', '2', '3'],
meta=dict(chp=True)))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp",
sequence=['1', '2', '3'],
meta=dict(ohp=True)))
cs.add_hyperparameter(Constant("const", value=1,
meta=dict(const=True)))
parent = ConfigurationSpace()
parent.add_configuration_space("sub", cs, delimiter=':')
self.assertEqual(
parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:chp").meta, dict(chp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True))
self.assertEqual(
parent.get_hyperparameter("sub:const").meta, dict(const=True))
def test_subspace_switches(self):
# create a switch to select one of two algorithms
algo_switch = CategoricalHyperparameter(
name="switch",
choices=["algo1", "algo2"],
weights=[0.25, 0.75],
default_value="algo1"
)
# create sub-configuration space for algorithm 1
algo1_cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter(name="algo1_param1", choices=["A", "B"], weights=[0.3, 0.7], default_value="B")
algo1_cs.add_hyperparameter(hp1)
# create sub-configuration space for algorithm 2
algo2_cs = ConfigurationSpace()
hp2 = CategoricalHyperparameter(name="algo2_param1", choices=["X", "Y"], default_value="Y")
algo2_cs.add_hyperparameter(hp2)
# create a configuration space and populate it with both the switch and the two sub-configuration spaces
cs = ConfigurationSpace()
cs.add_hyperparameter(algo_switch)
cs.add_configuration_space(
prefix="algo1_subspace",
configuration_space=algo1_cs,
parent_hyperparameter={'parent': algo_switch, 'value': "algo1"}
)
cs.add_configuration_space(
prefix="algo2_subspace",
configuration_space=algo2_cs,
parent_hyperparameter={'parent': algo_switch, 'value': "algo2"}
)
# check choices in the final configuration space
self.assertEqual(("algo1", "algo2"), cs.get_hyperparameter("switch").choices)
self.assertEqual(("A", "B"), cs.get_hyperparameter("algo1_subspace:algo1_param1").choices)
self.assertEqual(("X", "Y"), cs.get_hyperparameter("algo2_subspace:algo2_param1").choices)
# check probabilities in the final configuration space
self.assertEqual((0.25, 0.75), cs.get_hyperparameter("switch").probabilities)
self.assertEqual((0.3, 0.7), cs.get_hyperparameter("algo1_subspace:algo1_param1").probabilities)
self.assertEqual(None, cs.get_hyperparameter("algo2_subspace:algo2_param1").probabilities)
# check default values in the final configuration space
self.assertEqual("algo1", cs.get_hyperparameter("switch").default_value)
self.assertEqual("B", cs.get_hyperparameter("algo1_subspace:algo1_param1").default_value)
self.assertEqual("Y", cs.get_hyperparameter("algo2_subspace:algo2_param1").default_value)
def extract_hyperparameter(
eval_config: Dict[str, Any],
config_space: CS.ConfigurationSpace,
searching_space: Dict[str, ParameterSettings],
hp_module_path: str,
) -> Dict[str, Any]:
"""
Extract categorical values from another module
Args:
eval_config (Dict[str, Any]):
The configuration to evaluate.
The categorical is provided by one of the key strings.
config_space (CS.ConfigurationSpace):
The configuration space
searching_space (Dict[str, ParameterSettings]):
Dict information taken from a prepared json file.
hp_module_path (str):
The path where the `hyperparameters.py` for the target
objective exists.
Returns:
return_config (Dict[str, Any]):
The configuration that is created by replacing
all the categorical keys to the object of interests
such as function
"""
return_config = {}
hp_module = get_hyperparameter_module(hp_module_path)
for key, val in eval_config.items():
hp_info = searching_space[key]
if not isinstance(val, str):
return_config[key] = val
continue
if 'dataclass' in hp_info.keys():
choices_class_name = hp_info['dataclass']
assert (isinstance(choices_class_name, str))
choices = getattr(hp_module, choices_class_name)
return_config[key] = getattr(choices, val)
elif hasattr(config_space.get_hyperparameter(key), 'choices'):
return_config[key] = eval(val)
return return_config
def get_random_sample(hp_name: str, is_categorical: bool, is_ordinal: bool,
rng: np.random.RandomState,
config_space: CS.ConfigurationSpace) -> NumericType:
"""
Random sample of a provided hyperparameter
Args:
config_space (CS.ConfigurationSpace): The searching space information
is_categorical (bool): Whether this hyperparameter is categorical
is_ordinal (bool): Whether this hyperparameter is ordinal
hp_name (str): The names of a hyperparameter
rng (np.random.RandomState): The random state for numpy
Returns:
value (NumericType): A sampled value before conversion
* categorical -> index of a symbol
* log -> a value in a log-scale
* q -> no quantization here
"""
config = config_space.get_hyperparameter(hp_name)
if is_categorical:
choices = config.choices
sample = rng.randint(len(choices))
elif is_ordinal:
if config.meta is None:
raise ValueError(
'The meta information of the ordinal hyperparameter must be provided'
)
log = config.meta.get('log', False)
seq = config.sequence
sample = seq[rng.randint(len(seq))]
sample = np.log(sample) if log else sample
else:
lb = np.log(config.lower) if config.log else config.lower
ub = np.log(config.upper) if config.log else config.upper
sample = rng.uniform() * (ub - lb) + lb
return sample
def test_add_configuration_space_probabilities(self):
from copy import copy, deepcopy
from pickle import dumps, loads
weights = [0.25, 0.5, 0.25]
hp = CategoricalHyperparameter("B", ["1", "2", "3"], weights=weights)
sub_cs = ConfigurationSpace()
sub_cs.add_hyperparameter(hp)
cs = ConfigurationSpace()
cs.add_configuration_space("A", sub_cs)
self.assertEqual(
deepcopy(sub_cs).get_hyperparameter("B").probabilities, weights)
self.assertEqual(
copy(sub_cs).get_hyperparameter("B").probabilities, weights)
self.assertEqual(
loads(dumps(sub_cs)).get_hyperparameter("B").probabilities,
weights)
self.assertEqual(cs.get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
deepcopy(cs).get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
copy(cs).get_hyperparameter("A:B").probabilities, weights)
self.assertEqual(
loads(dumps(cs)).get_hyperparameter("A:B").probabilities, weights)
def __init__(self,
node_list,
node_index,
task_type,
timestamp,
fe_config_space: ConfigurationSpace,
cash_config_space: ConfigurationSpace,
data: DataNode,
fixed_config=None,
time_limit=None,
trial_num=0,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=50,
per_run_time_limit=300,
output_dir="logs",
dataset_name='default_dataset',
eval_type='holdout',
resampling_params=None,
n_jobs=1,
seed=1):
"""
:param classifier_ids: subset of {'adaboost','bernoulli_nb','decision_tree','extra_trees','gaussian_nb','gradient_boosting',
'gradient_boosting','k_nearest_neighbors','lda','liblinear_svc','libsvm_svc','multinomial_nb','passive_aggressive','qda',
'random_forest','sgd'}
"""
super(ConditioningBlock,
self).__init__(node_list,
node_index,
task_type,
timestamp,
fe_config_space,
cash_config_space,
data,
fixed_config=fixed_config,
time_limit=time_limit,
trial_num=trial_num,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
# Best configuration.
self.optimal_arm = None
self.best_lower_bounds = None
# Bandit settings.
self.alpha = 4
self.arms = list(
cash_config_space.get_hyperparameter('algorithm').choices)
self.rewards = dict()
self.sub_bandits = dict()
self.evaluation_cost = dict()
self.arm_cost_stats = dict()
for _arm in self.arms:
self.arm_cost_stats[_arm] = list()
for arm in self.arms:
self.rewards[arm] = list()
self.evaluation_cost[arm] = list()
hps = cash_config_space.get_hyperparameters()
cs = ConfigurationSpace()
cs.add_hyperparameter(Constant('algorithm', arm))
for hp in hps:
if hp.name.split(':')[0] == arm:
cs.add_hyperparameter(hp)
# Add active conditions
conds = cash_config_space.get_conditions()
for cond in conds:
try:
cs.add_condition(cond)
except:
pass
# Add active forbidden clauses
forbids = cash_config_space.get_forbiddens()
for forbid in forbids:
try:
cs.add_forbidden_clause(forbid)
except:
pass
from solnml.blocks.block_utils import get_node_type
child_type = get_node_type(node_list, node_index + 1)
self.sub_bandits[arm] = child_type(
node_list,
node_index + 1,
task_type,
timestamp,
deepcopy(fe_config_space),
deepcopy(cs),
data.copy_(),
fixed_config=fixed_config,
time_limit=time_limit,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
self.action_sequence = list()
self.final_rewards = list()
self.start_time = time.time()
self.time_records = list()
# Initialize the parameters.
self.pull_cnt = 0
self.pick_id = 0
self.update_cnt = 0
arm_num = len(self.arms)
self.optimal_algo_id = None
self.arm_candidate = self.arms.copy()
self.best_lower_bounds = np.zeros(arm_num)
_iter_id = 0
if self.time_limit is None:
if arm_num * self.alpha > self.trial_num:
raise ValueError('Trial number should be larger than %d.' %
(arm_num * self.alpha))
else:
self.trial_num = MAX_INT
def get_hyperparameter_search_space(dataset_properties=None):
max_num_layers = 7 # Maximum number of layers coded
# Hacky way to condition layers params based on the number of layers
# 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer
layer_choices = [
chr(i) for i in range(ord('c'),
ord('b') + max_num_layers)
]
batch_size = UniformIntegerHyperparameter("batch_size",
32,
4096,
log=True,
default=32)
number_epochs = UniformIntegerHyperparameter("number_epochs",
2,
80,
default=5)
num_layers = CategoricalHyperparameter("num_layers",
choices=layer_choices,
default='c')
lr = UniformFloatHyperparameter("learning_rate",
1e-6,
1.0,
log=True,
default=0.01)
l2 = UniformFloatHyperparameter("lambda2",
1e-7,
1e-2,
log=True,
default=1e-4)
dropout_output = UniformFloatHyperparameter("dropout_output",
0.0,
0.99,
default=0.5)
# Define basic hyperparameters and define the config space
# basic means that are independent from the number of layers
cs = ConfigurationSpace()
cs.add_hyperparameter(number_epochs)
cs.add_hyperparameter(batch_size)
cs.add_hyperparameter(num_layers)
cs.add_hyperparameter(lr)
cs.add_hyperparameter(l2)
cs.add_hyperparameter(dropout_output)
# Define parameters with different child parameters and conditions
solver_choices = [
"adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov",
"smorm3s"
]
solver = CategoricalHyperparameter(name="solver",
choices=solver_choices,
default="smorm3s")
beta1 = UniformFloatHyperparameter("beta1",
1e-4,
0.1,
log=True,
default=0.1)
beta2 = UniformFloatHyperparameter("beta2",
1e-4,
0.1,
log=True,
default=0.01)
rho = UniformFloatHyperparameter("rho",
0.05,
0.99,
log=True,
default=0.95)
momentum = UniformFloatHyperparameter("momentum",
0.3,
0.999,
default=0.9)
# TODO: Add policy based on this sklearn sgd
policy_choices = ['fixed', 'inv', 'exp', 'step']
lr_policy = CategoricalHyperparameter(name="lr_policy",
choices=policy_choices,
default='fixed')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-3,
upper=1e-1,
default=1e-2)
power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5)
epoch_step = UniformIntegerHyperparameter("epoch_step",
2,
20,
default=5)
cs.add_hyperparameter(solver)
cs.add_hyperparameter(beta1)
cs.add_hyperparameter(beta2)
cs.add_hyperparameter(momentum)
cs.add_hyperparameter(rho)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(power)
cs.add_hyperparameter(epoch_step)
# Define parameters that are needed it for each layer
output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh']
activations_choices = [
'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear'
]
weight_choices = [
'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform',
'he_normal', 'he_uniform', 'ortogonal', 'sparse'
]
# Iterate over parameters that are used in each layer
for i in range(1, max_num_layers):
layer_units = UniformIntegerHyperparameter("num_units_layer_" +
str(i),
64,
4096,
log=True,
default=128)
cs.add_hyperparameter(layer_units)
layer_dropout = UniformFloatHyperparameter("dropout_layer_" +
str(i),
0.0,
0.99,
default=0.5)
cs.add_hyperparameter(layer_dropout)
weight_initialization = CategoricalHyperparameter(
'weight_init_' + str(i),
choices=weight_choices,
default='he_normal')
cs.add_hyperparameter(weight_initialization)
layer_std = UniformFloatHyperparameter("std_layer_" + str(i),
1e-6,
0.1,
log=True,
default=0.005)
cs.add_hyperparameter(layer_std)
layer_activation = CategoricalHyperparameter(
"activation_layer_" + str(i),
choices=activations_choices,
default="relu")
cs.add_hyperparameter(layer_activation)
layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' +
str(i),
0.01,
0.99,
default=0.3)
cs.add_hyperparameter(layer_leakiness)
layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' +
str(i),
0.5,
1.0,
default=2. / 3.)
cs.add_hyperparameter(layer_tanh_alpha)
layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' +
str(i),
1.1,
3.0,
log=True,
default=1.7159)
cs.add_hyperparameter(layer_tanh_beta)
# TODO: Could be in a function in a new module
for i in range(2, max_num_layers):
# Condition layers parameter on layer choice
layer_unit_param = cs.get_hyperparameter("num_units_layer_" +
str(i))
layer_cond = InCondition(child=layer_unit_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition dropout parameter on layer choice
layer_dropout_param = cs.get_hyperparameter("dropout_layer_" +
str(i))
layer_cond = InCondition(child=layer_dropout_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition weight initialization on layer choice
layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i))
layer_cond = InCondition(child=layer_weight_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition std parameter on weight layer initialization choice
layer_std_param = cs.get_hyperparameter("std_layer_" + str(i))
weight_cond = EqualsCondition(child=layer_std_param,
parent=layer_weight_param,
value='normal')
cs.add_condition(weight_cond)
# Condition activation parameter on layer choice
layer_activation_param = cs.get_hyperparameter(
"activation_layer_" + str(i))
layer_cond = InCondition(child=layer_activation_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition leakiness on activation choice
layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_leakiness_param,
parent=layer_activation_param,
value='leaky')
cs.add_condition(activation_cond)
# Condition tanh on activation choice
layer_tanh_alpha_param = cs.get_hyperparameter(
"tanh_alpha_layer_" + str(i))
activation_cond = EqualsCondition(child=layer_tanh_alpha_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_tanh_beta_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
# Conditioning on solver
momentum_depends_on_solver = InCondition(
momentum, solver, values=["momentum", "nesterov"])
beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam")
beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam")
rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta")
cs.add_condition(momentum_depends_on_solver)
cs.add_condition(beta1_depends_on_solver)
cs.add_condition(beta2_depends_on_solver)
cs.add_condition(rho_depends_on_solver)
# Conditioning on learning rate policy
lr_policy_depends_on_solver = InCondition(
lr_policy, solver,
["adadelta", "adagrad", "sgd", "momentum", "nesterov"])
gamma_depends_on_policy = InCondition(child=gamma,
parent=lr_policy,
values=["inv", "exp", "step"])
power_depends_on_policy = EqualsCondition(power, lr_policy, "inv")
epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy,
"step")
cs.add_condition(lr_policy_depends_on_solver)
cs.add_condition(gamma_depends_on_policy)
cs.add_condition(power_depends_on_policy)
cs.add_condition(epoch_step_depends_on_policy)
return cs
def __rely_model(self, cs: ConfigurationSpace):
if not RelyModels.info:
return
all_models = list(
cs.get_hyperparameter(f"{PHASE2}:__choice__").choices)
rely_model_counter = Counter([x[0] for x in RelyModels.info])
# 依赖模式->所有相应模型
relied2AllModels = {}
# 依赖模式->无交集相应模型
relied2models = {}
for rely_model in rely_model_counter.keys():
_, hit = self.get_forbid_hit_in_models_by_rely(
all_models, rely_model)
relied2AllModels[rely_model] = hit
# 如果某依赖模式不对应任何模型，删除
for k, v in list(relied2AllModels.items()):
if not v:
relied2AllModels.pop(k)
rely_model_counter.pop(k)
has_any_hit = any(relied2AllModels.values())
if not has_any_hit:
return
# 按照规则计算 relied2models ： 无交集相应模型
relied_cnts_tuples = [(k, v) for k, v in rely_model_counter.items()]
relied_cnts_tuples.sort(key=lambda x: x[-1])
visited = set()
for rely_model, _ in relied_cnts_tuples:
models = relied2AllModels[rely_model]
for other in set(rely_model_counter.keys()) - {rely_model}:
if (rely_model, other) in visited:
continue
other_models = relied2AllModels[other]
if len(other_models) <= len(models):
models = list(set(models) - set(other_models))
visited.add((rely_model, other))
visited.add((other, rely_model))
relied2models[rely_model] = models
# 键的顺序遵循rely_model_counter.keys()
def objective(relyModel2prob, debug=False):
# relyModel2prob = {rely_model: prob for rely_model, prob in zip(list(rely_model_counter.keys()), args)}
cur_cs = deepcopy(cs)
self.set_probabilities_in_cs(cur_cs, relied2models,
relied2AllModels, all_models,
**relyModel2prob)
cur_cs.seed(42)
try:
sample_times = len(all_models) * 15
counter = Counter([
_hp.get(f"{PHASE2}:__choice__")
for _hp in cur_cs.sample_configuration(sample_times)
])
if debug:
self.logger.info(
f"Finally, sample {sample_times} times in estimator list's frequency: \n{counter}"
)
except Exception:
return np.inf
vl = list(counter.values())
return np.var(vl) + 100 * (len(models) - len(vl))
space = {}
eps = 0.001
N_rely_model = len(rely_model_counter.keys())
for rely_model in rely_model_counter.keys():
space[rely_model] = hp.uniform(rely_model, eps,
(1 / N_rely_model) - eps)
best = fmin(
fn=objective,
space=space,
algo=tpe.suggest,
max_evals=100,
rstate=np.random.RandomState(42),
show_progressbar=False,
)
self.logger.info(f"The best probability is {best}")
objective(best, debug=True)
self.set_probabilities_in_cs(cs, relied2models, relied2AllModels,
all_models, **best)
def run_on_tasks(config_frame_orig: pd.DataFrame,
surrogates: typing.Dict[int, sklearn.pipeline.Pipeline],
quality_frame: pd.DataFrame,
config_space: ConfigSpace.ConfigurationSpace,
search_hyperparameters: typing.List[str],
search_transform_fns: typing.List[str],
hold_out_task: typing.Optional[int], resized_grid_size: int,
output_file: str):
hold_out_surrogate = None
if hold_out_task is not None:
hold_out_surrogate = surrogates[hold_out_task]
surrogates = dict(surrogates)
del surrogates[hold_out_task]
# performance untransformed
baseline_configuration, baseline_results_per_task = select_best_configuration_across_tasks(
config_frame_orig, surrogates, config_frame_orig.columns.values, None,
None, None, None)
baseline_avg_performance = np.average(baseline_results_per_task)
baseline_holdout = None
baseline_random_search = None
if hold_out_task is not None:
baseline_holdout = openmldefaults.utils.single_prediction(
config_frame_orig, hold_out_surrogate, baseline_configuration)
baseline_random_search = [
openmldefaults.utils.single_prediction(
config_frame_orig, hold_out_surrogate,
config_space.sample_configuration(1).get_dictionary())
for i in range(50)
]
logging.info('Baseline: %s [%s] %s. Holdout task: %s' %
(baseline_configuration, baseline_results_per_task,
baseline_avg_performance, baseline_holdout))
transform_fns = openmldefaults.symbolic.all_transform_fns()
search_transform_fns = search_transform_fns if search_transform_fns is not None else transform_fns.keys(
)
search_hyperparameters = search_hyperparameters if search_hyperparameters is not None \
else [hp.name for hp in config_space.get_hyperparameters()]
symbolic_defaults = list()
for idx_hp, hyperparameter_name in enumerate(search_hyperparameters):
hyperparameter = config_space.get_hyperparameter(hyperparameter_name)
if isinstance(hyperparameter, ConfigSpace.hyperparameters.Constant):
logging.warning('Skipping Constant Hyperparameter: %s' %
hyperparameter.name)
continue
if isinstance(
hyperparameter,
ConfigSpace.hyperparameters.UnParametrizedHyperparameter):
logging.warning('Skipping Unparameterized Hyperparameter: %s' %
hyperparameter.name)
continue
if not isinstance(hyperparameter,
ConfigSpace.hyperparameters.NumericalHyperparameter):
logging.warning('Skipping Non-Numerical Hyperparameter: %s' %
hyperparameter.name)
continue
logging.info(
'Started with hyperparameter %s (%d/%d)' %
(hyperparameter.name, idx_hp + 1, len(search_hyperparameters)))
config_space_prime = openmldefaults.utils.remove_hyperparameter(
config_space, hyperparameter.name)
configurations = openmldefaults.utils.generate_grid_configurations(
config_space_prime, 0, resized_grid_size)
config_frame_prime = pd.DataFrame(configurations)
for idx_trnfm_fn, transform_name in enumerate(search_transform_fns):
logging.info(
'- Transformer fn %s (%d/%d)' %
(transform_name, idx_trnfm_fn + 1, len(transform_fns)))
geom_space = np.geomspace(0.01, 2, 10)
geom_space = np.append(geom_space, [1])
for idx_av, alpha_value in enumerate(geom_space):
logging.info('--- Alpha value %f (%d/%d)' %
(alpha_value, idx_av + 1, len(geom_space)))
for meta_feature in quality_frame.columns.values:
try:
transform_fn = openmldefaults.symbolic.all_transform_fns(
)[transform_name]
symbolic_config, symbolic_results_per_task = select_best_configuration_across_tasks(
config_frame_prime,
surrogates,
config_frame_orig.columns.
values, # note to take the original frame
hyperparameter.name,
transform_fn,
alpha_value,
quality_frame[meta_feature].to_dict(),
)
symbolic_average_performance = np.average(
symbolic_results_per_task)
if symbolic_average_performance > baseline_avg_performance:
symbolic_holdout_score = None
if hold_out_surrogate is not None:
symbolic_value = transform_fn(
alpha_value,
quality_frame[meta_feature][hold_out_task])
symbolic_config[
hyperparameter.name] = symbolic_value
symbolic_holdout_score = openmldefaults.utils.single_prediction(
config_frame_orig, hold_out_surrogate,
symbolic_config)
current_result = {
'configuration': symbolic_config,
'results_per_task': symbolic_results_per_task,
'avg_performance':
symbolic_average_performance,
'holdout_score': symbolic_holdout_score,
'trasnform_hyperparameter':
hyperparameter.name,
'transform_fn': transform_name,
'transform_alpha_value': alpha_value,
'transform_meta_feature': meta_feature,
}
symbolic_defaults.append(current_result)
logging.info(
'Found improvement over base-line: %s' %
current_result)
except ZeroDivisionError:
logging.warning(
'Zero division error with (fn=%s, alpha=%s, meta_f=%s). '
'skipping. ' %
(transform_name, alpha_value, meta_feature))
pass
except OverflowError:
logging.warning(
'Overflow error with (fn=%s, alpha=%s, meta_f=%s). '
'skipping. ' %
(transform_name, alpha_value, meta_feature))
pass
except ValueError:
# keep a close eye on this one. Question: why do the others not catch this one?
logging.warning(
'Overflow error with (fn=%s, alpha=%s, meta_f=%s). '
'skipping. ' %
(transform_name, alpha_value, meta_feature))
pass
total = {
'baseline_configuration': baseline_configuration,
'baseline_avg_performance': baseline_avg_performance,
'baseline_random_search': baseline_random_search,
'baseline_results_per_task': baseline_results_per_task,
'baseline_holdout_score': baseline_holdout,
'symbolic_defaults': symbolic_defaults
}
with open(output_file, 'wb') as fp:
pickle.dump(obj=total, file=fp, protocol=0)
logging.info('Saved result file to: %s' % output_file)