def test_fix_types(self): # Test categorical and ordinal for hyperparameter_type in [CategoricalHyperparameter, OrdinalHyperparameter]: cs = ConfigurationSpace() cs.add_hyperparameters([ hyperparameter_type('bools', [True, False]), hyperparameter_type('ints', [1, 2, 3, 4, 5]), hyperparameter_type('floats', [1.5, 2.5, 3.5, 4.5, 5.5]), hyperparameter_type('str', ['string', 'ding', 'dong']), hyperparameter_type('mixed', [2, True, 1.5, 'string', False, 'False']), ]) c = cs.get_default_configuration().get_dictionary() # Check bools for b in [False, True]: c['bools'] = b c_str = {k: str(v) for k, v in c.items()} self.assertEqual(fix_types(c_str, cs), c) # Check legal mixed values for m in [2, True, 1.5, 'string']: c['mixed'] = m c_str = {k: str(v) for k, v in c.items()} self.assertEqual(fix_types(c_str, cs), c) # Check error on cornercase that cannot be caught for m in [False, 'False']: c['mixed'] = m c_str = {k: str(v) for k, v in c.items()} self.assertRaises(ValueError, fix_types, c_str, cs) # Test constant for m in [2, 1.5, 'string']: cs = ConfigurationSpace() cs.add_hyperparameter(Constant('constant', m)) c = cs.get_default_configuration().get_dictionary() c_str = {k: str(v) for k, v in c.items()} self.assertEqual(fix_types(c_str, cs), c)
def sample_configurations(configuration_space: ConfigurationSpace, sample_size: int, historical_configs: List[Configuration], seed=1): configuration_space.seed(seed) result = [] sample_cnt = 0 if len(historical_configs) == 0: result.append(configuration_space.get_default_configuration()) while len(result) < sample_size: config = configuration_space.sample_configuration(1) if config not in result and config not in historical_configs: result.append(config) sample_cnt += 1 if sample_cnt > 50 * sample_size: break # if len(result) == 0: # hist_num = len(historical_configs) # if hist_num > sample_size: # idxs = random.sample(range(len(historical_configs)), sample_size) # result = [historical_configs[idx] for idx in idxs] # else: # result = historical_configs.copy() return result
def test_setitem(self): ''' Checks overriding a sampled configuration ''' pcs = ConfigurationSpace() pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1)) x1 = pcs.add_hyperparameter( CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab') ) # Condition x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2])) pcs.add_condition(EqualsCondition(x2, x1, 'ab')) # Forbidden x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2])) pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2)) conf = pcs.get_default_configuration() # failed because it's a invalid configuration with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"): conf['x0'] = 0 # failed because the variable didn't exists with self.assertRaisesRegex( KeyError, "Hyperparameter 'x_0' does not exist in this configuration space.", ): conf['x_0'] = 1 # failed because forbidden clause is violated with self.assertRaisesRegex( ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2", ): conf['x3'] = 2 self.assertEqual(conf['x3'], 1) # successful operation 1 x0_old = conf['x0'] if x0_old == 1: conf['x0'] = 2 else: conf['x0'] = 1 x0_new = conf['x0'] self.assertNotEqual(x0_old, x0_new) pcs._check_configuration_rigorous(conf) self.assertEqual(conf['x2'], 1) # successful operation 2 x1_old = conf['x1'] if x1_old == 'ab': conf['x1'] = 'cd' else: conf['x1'] = 'ab' x1_new = conf['x1'] self.assertNotEqual(x1_old, x1_new) pcs._check_configuration_rigorous(conf) self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_setitem(self): ''' Checks overriding a sampled configuration ''' pcs = ConfigurationSpace() pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1)) x1 = pcs.add_hyperparameter(CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab')) # Condition x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2])) pcs.add_condition(EqualsCondition(x2, x1, 'ab')) # Forbidden x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2])) pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2)) conf = pcs.get_default_configuration() # failed because it's a invalid configuration with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"): conf['x0'] = 0 # failed because the variable didn't exists with self.assertRaisesRegex(KeyError, "Hyperparameter 'x_0' does not exist in this configuration space."): conf['x_0'] = 1 # failed because forbidden clause is violated with self.assertRaisesRegex(ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"): conf['x3'] = 2 self.assertEqual(conf['x3'], 1) # successful operation 1 x0_old = conf['x0'] if x0_old == 1: conf['x0'] = 2 else: conf['x0'] = 1 x0_new = conf['x0'] self.assertNotEqual(x0_old, x0_new) pcs._check_configuration_rigorous(conf) self.assertEqual(conf['x2'], 1) # successful operation 2 x1_old = conf['x1'] if x1_old == 'ab': conf['x1'] = 'cd' else: conf['x1'] = 'ab' x1_new = conf['x1'] self.assertNotEqual(x1_old, x1_new) pcs._check_configuration_rigorous(conf) self.assertRaises(KeyError, conf.__getitem__, 'x2')
def sample_configurations(configuration_space: ConfigurationSpace, historical_configs: List[Configuration], sample_size: int): result = list() sample_cnt = 0 if len(historical_configs) == 0: result.append(configuration_space.get_default_configuration()) while len(result) < sample_size: config = configuration_space.sample_configuration(1) if config not in result and config not in historical_configs: result.append(config) sample_cnt += 1 if sample_cnt > 50 * sample_size: result.append(config) break return result
def test_repr_roundtrip(self): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("uihp", lower=1, upper=10)) cs.add_hyperparameter( NormalIntegerHyperparameter("nihp", mu=0, sigma=1)) cs.add_hyperparameter( UniformFloatHyperparameter("ufhp", lower=1, upper=10)) cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1)) cs.add_hyperparameter( CategoricalHyperparameter("chp", choices=['1', '2', '3'])) cs.add_hyperparameter( OrdinalHyperparameter("ohp", sequence=['1', '2', '3'])) cs.add_hyperparameter(Constant("const", value=1)) default = cs.get_default_configuration() repr = default.__repr__() repr = repr.replace('})', '}, configuration_space=cs)') config = eval(repr) self.assertEqual(default, config)
def _test_get_one_exchange_neighbourhood(self, hp): cs = ConfigurationSpace() num_neighbors = 0 if not isinstance(hp, list): hp = [hp] for hp_ in hp: cs.add_hyperparameter(hp_) if np.isinf(hp_.get_num_neighbors()): num_neighbors += 4 else: num_neighbors += hp_.get_num_neighbors() cs.seed(1) config = cs.get_default_configuration() all_neighbors = [] for i in range(100): neighborhood = get_one_exchange_neighbourhood(config, i) for new_config in neighborhood: self.assertNotEqual(config, new_config) all_neighbors.append(new_config) return all_neighbors
def _check_and_cast_fidelity(fidelity: Union[dict, ConfigSpace.Configuration, None], fidelity_space: ConfigSpace.ConfigurationSpace, **kwargs) \ -> ConfigSpace.Configuration: """ Helper-function to evaluate the given fidelity object. Similar to the checking and casting from above, we validate the fidelity object. To do so, we cast it to a ConfigSpace.Configuration object. If the fidelity is not specified (None), then we use the default fidelity of the benchmark. If the benchmark is a multi-multi-fidelity benchmark and only a subset of the available fidelities is specified, we fill the missing ones with their default values. """ # Make a check, that no fidelities are in the kwargs. f_in_kwargs = [] for f in fidelity_space.get_hyperparameters(): if f.name in kwargs: f_in_kwargs.append(f.name) if len(f_in_kwargs) != 0: raise ValueError(f'Fidelity parameters {", ".join(f_in_kwargs)} should not be part of kwargs\n' f'Fidelity: {fidelity}\n Kwargs: {kwargs}') default_fidelities = fidelity_space.get_default_configuration() if fidelity is None: fidelity = default_fidelities if isinstance(fidelity, dict): default_fidelities_cfg = default_fidelities.get_dictionary() fidelity_copy = fidelity.copy() fidelity = {k: fidelity_copy.pop(k, v) for k, v in default_fidelities_cfg.items()} assert len(fidelity_copy) == 0, 'Provided fidelity dict contained unknown fidelity ' \ f'values: {fidelity_copy.keys()}' fidelity = ConfigSpace.Configuration(fidelity_space, fidelity) elif isinstance(fidelity, ConfigSpace.Configuration): fidelity = fidelity else: raise TypeError(f'Fidelity has to be an instance of type None, dict, or ' f'ConfigSpace.Configuration but was {type(fidelity)}') # Ensure that the extracted fidelity values play well with the defined fidelity space fidelity_space.check_configuration(fidelity) return fidelity
def __init__(self, node_list, node_index, task_type, timestamp, fe_config_space: ConfigurationSpace, cash_config_space: ConfigurationSpace, data: DataNode, fixed_config=None, time_limit=None, trial_num=0, metric='acc', ensemble_method='ensemble_selection', ensemble_size=50, per_run_time_limit=300, output_dir="logs", dataset_name='default_dataset', eval_type='holdout', resampling_params=None, n_jobs=1, seed=1): super(AlternatingBlock, self).__init__(node_list, node_index, task_type, timestamp, fe_config_space, cash_config_space, data, fixed_config=fixed_config, time_limit=time_limit, trial_num=trial_num, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) self.arms = ['hpo', 'fe'] self.optimal_algo_id = None self.first_start = True self.sub_bandits = dict() self.rewards = dict() self.evaluation_cost = dict() self.update_flag = dict() # Global incumbent. self.init_config = { 'fe': fe_config_space.get_default_configuration().get_dictionary().copy( ), 'hpo': cash_config_space.get_default_configuration().get_dictionary(). copy() } self.inc = { 'fe': fe_config_space.get_default_configuration().get_dictionary().copy( ), 'hpo': cash_config_space.get_default_configuration().get_dictionary(). copy() } self.local_inc = { 'fe': fe_config_space.get_default_configuration().get_dictionary().copy( ), 'hpo': cash_config_space.get_default_configuration().get_dictionary(). copy() } self.local_hist = {'fe': [], 'hpo': []} self.inc_record = {'fe': list(), 'hpo': list()} self.exp_output = dict() self.eval_dict = dict() self.arm_eval_dict = {'fe': dict(), 'hpo': dict()} for arm in self.arms: self.rewards[arm] = list() self.update_flag[arm] = False self.evaluation_cost[arm] = list() self.exp_output[arm] = dict() self.pull_cnt = 0 self.action_sequence = list() self.final_rewards = list() for arm in self.arms: if arm == 'fe': from solnml.blocks.block_utils import get_node_type child_type = get_node_type(node_list, node_index + 1) self.sub_bandits[arm] = child_type( node_list, node_index + 1, task_type, timestamp, fe_config_space, None, data.copy_(), fixed_config=self.init_config['hpo'], time_limit=time_limit, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) else: from solnml.blocks.block_utils import get_node_type child_type = get_node_type(node_list, node_index + 2) self.sub_bandits[arm] = child_type( node_list, node_index + 2, task_type, timestamp, None, cash_config_space, data.copy_(), fixed_config=self.init_config['fe'], time_limit=time_limit, metric=metric, ensemble_method=ensemble_method, ensemble_size=ensemble_size, per_run_time_limit=per_run_time_limit, output_dir=output_dir, dataset_name=dataset_name, eval_type=eval_type, resampling_params=resampling_params, n_jobs=n_jobs, seed=seed) self.topk_saver = CombinedTopKModelSaver(k=50, model_dir=self.output_dir, identifier=self.timestamp)