def test_fix_types(self):
# Test categorical and ordinal
for hyperparameter_type in [CategoricalHyperparameter, OrdinalHyperparameter]:
cs = ConfigurationSpace()
cs.add_hyperparameters([
hyperparameter_type('bools', [True, False]),
hyperparameter_type('ints', [1, 2, 3, 4, 5]),
hyperparameter_type('floats', [1.5, 2.5, 3.5, 4.5, 5.5]),
hyperparameter_type('str', ['string', 'ding', 'dong']),
hyperparameter_type('mixed', [2, True, 1.5, 'string', False, 'False']),
])
c = cs.get_default_configuration().get_dictionary()
# Check bools
for b in [False, True]:
c['bools'] = b
c_str = {k: str(v) for k, v in c.items()}
self.assertEqual(fix_types(c_str, cs), c)
# Check legal mixed values
for m in [2, True, 1.5, 'string']:
c['mixed'] = m
c_str = {k: str(v) for k, v in c.items()}
self.assertEqual(fix_types(c_str, cs), c)
# Check error on cornercase that cannot be caught
for m in [False, 'False']:
c['mixed'] = m
c_str = {k: str(v) for k, v in c.items()}
self.assertRaises(ValueError, fix_types, c_str, cs)
# Test constant
for m in [2, 1.5, 'string']:
cs = ConfigurationSpace()
cs.add_hyperparameter(Constant('constant', m))
c = cs.get_default_configuration().get_dictionary()
c_str = {k: str(v) for k, v in c.items()}
self.assertEqual(fix_types(c_str, cs), c)
def sample_configurations(configuration_space: ConfigurationSpace,
sample_size: int,
historical_configs: List[Configuration],
seed=1):
configuration_space.seed(seed)
result = []
sample_cnt = 0
if len(historical_configs) == 0:
result.append(configuration_space.get_default_configuration())
while len(result) < sample_size:
config = configuration_space.sample_configuration(1)
if config not in result and config not in historical_configs:
result.append(config)
sample_cnt += 1
if sample_cnt > 50 * sample_size:
break
# if len(result) == 0:
# hist_num = len(historical_configs)
# if hist_num > sample_size:
# idxs = random.sample(range(len(historical_configs)), sample_size)
# result = [historical_configs[idx] for idx in idxs]
# else:
# result = historical_configs.copy()
return result
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(
CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab')
)
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space.",
):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(
ForbiddenValueError,
"Given vector violates forbidden clause Forbidden: x3 == 2",
):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(KeyError, "Hyperparameter 'x_0' does not exist in this configuration space."):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def sample_configurations(configuration_space: ConfigurationSpace,
historical_configs: List[Configuration],
sample_size: int):
result = list()
sample_cnt = 0
if len(historical_configs) == 0:
result.append(configuration_space.get_default_configuration())
while len(result) < sample_size:
config = configuration_space.sample_configuration(1)
if config not in result and config not in historical_configs:
result.append(config)
sample_cnt += 1
if sample_cnt > 50 * sample_size:
result.append(config)
break
return result
def test_repr_roundtrip(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp", lower=1, upper=10))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp", mu=0, sigma=1))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp", lower=1, upper=10))
cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1))
cs.add_hyperparameter(
CategoricalHyperparameter("chp", choices=['1', '2', '3']))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp", sequence=['1', '2', '3']))
cs.add_hyperparameter(Constant("const", value=1))
default = cs.get_default_configuration()
repr = default.__repr__()
repr = repr.replace('})', '}, configuration_space=cs)')
config = eval(repr)
self.assertEqual(default, config)
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors
def _check_and_cast_fidelity(fidelity: Union[dict, ConfigSpace.Configuration, None],
fidelity_space: ConfigSpace.ConfigurationSpace, **kwargs) \
-> ConfigSpace.Configuration:
""" Helper-function to evaluate the given fidelity object.
Similar to the checking and casting from above, we validate the fidelity object. To do so, we cast it to a
ConfigSpace.Configuration object.
If the fidelity is not specified (None), then we use the default fidelity of the benchmark.
If the benchmark is a multi-multi-fidelity benchmark and only a subset of the available fidelities is
specified, we fill the missing ones with their default values.
"""
# Make a check, that no fidelities are in the kwargs.
f_in_kwargs = []
for f in fidelity_space.get_hyperparameters():
if f.name in kwargs:
f_in_kwargs.append(f.name)
if len(f_in_kwargs) != 0:
raise ValueError(f'Fidelity parameters {", ".join(f_in_kwargs)} should not be part of kwargs\n'
f'Fidelity: {fidelity}\n Kwargs: {kwargs}')
default_fidelities = fidelity_space.get_default_configuration()
if fidelity is None:
fidelity = default_fidelities
if isinstance(fidelity, dict):
default_fidelities_cfg = default_fidelities.get_dictionary()
fidelity_copy = fidelity.copy()
fidelity = {k: fidelity_copy.pop(k, v) for k, v in default_fidelities_cfg.items()}
assert len(fidelity_copy) == 0, 'Provided fidelity dict contained unknown fidelity ' \
f'values: {fidelity_copy.keys()}'
fidelity = ConfigSpace.Configuration(fidelity_space, fidelity)
elif isinstance(fidelity, ConfigSpace.Configuration):
fidelity = fidelity
else:
raise TypeError(f'Fidelity has to be an instance of type None, dict, or '
f'ConfigSpace.Configuration but was {type(fidelity)}')
# Ensure that the extracted fidelity values play well with the defined fidelity space
fidelity_space.check_configuration(fidelity)
return fidelity
def __init__(self,
node_list,
node_index,
task_type,
timestamp,
fe_config_space: ConfigurationSpace,
cash_config_space: ConfigurationSpace,
data: DataNode,
fixed_config=None,
time_limit=None,
trial_num=0,
metric='acc',
ensemble_method='ensemble_selection',
ensemble_size=50,
per_run_time_limit=300,
output_dir="logs",
dataset_name='default_dataset',
eval_type='holdout',
resampling_params=None,
n_jobs=1,
seed=1):
super(AlternatingBlock,
self).__init__(node_list,
node_index,
task_type,
timestamp,
fe_config_space,
cash_config_space,
data,
fixed_config=fixed_config,
time_limit=time_limit,
trial_num=trial_num,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
self.arms = ['hpo', 'fe']
self.optimal_algo_id = None
self.first_start = True
self.sub_bandits = dict()
self.rewards = dict()
self.evaluation_cost = dict()
self.update_flag = dict()
# Global incumbent.
self.init_config = {
'fe':
fe_config_space.get_default_configuration().get_dictionary().copy(
),
'hpo':
cash_config_space.get_default_configuration().get_dictionary().
copy()
}
self.inc = {
'fe':
fe_config_space.get_default_configuration().get_dictionary().copy(
),
'hpo':
cash_config_space.get_default_configuration().get_dictionary().
copy()
}
self.local_inc = {
'fe':
fe_config_space.get_default_configuration().get_dictionary().copy(
),
'hpo':
cash_config_space.get_default_configuration().get_dictionary().
copy()
}
self.local_hist = {'fe': [], 'hpo': []}
self.inc_record = {'fe': list(), 'hpo': list()}
self.exp_output = dict()
self.eval_dict = dict()
self.arm_eval_dict = {'fe': dict(), 'hpo': dict()}
for arm in self.arms:
self.rewards[arm] = list()
self.update_flag[arm] = False
self.evaluation_cost[arm] = list()
self.exp_output[arm] = dict()
self.pull_cnt = 0
self.action_sequence = list()
self.final_rewards = list()
for arm in self.arms:
if arm == 'fe':
from solnml.blocks.block_utils import get_node_type
child_type = get_node_type(node_list, node_index + 1)
self.sub_bandits[arm] = child_type(
node_list,
node_index + 1,
task_type,
timestamp,
fe_config_space,
None,
data.copy_(),
fixed_config=self.init_config['hpo'],
time_limit=time_limit,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
else:
from solnml.blocks.block_utils import get_node_type
child_type = get_node_type(node_list, node_index + 2)
self.sub_bandits[arm] = child_type(
node_list,
node_index + 2,
task_type,
timestamp,
None,
cash_config_space,
data.copy_(),
fixed_config=self.init_config['fe'],
time_limit=time_limit,
metric=metric,
ensemble_method=ensemble_method,
ensemble_size=ensemble_size,
per_run_time_limit=per_run_time_limit,
output_dir=output_dir,
dataset_name=dataset_name,
eval_type=eval_type,
resampling_params=resampling_params,
n_jobs=n_jobs,
seed=seed)
self.topk_saver = CombinedTopKModelSaver(k=50,
model_dir=self.output_dir,
identifier=self.timestamp)