def set_hyperparameters( self, configuration: Configuration, init_params: Optional[Dict[str, Any]] = None) -> 'OHEChoice': new_params = {} params = configuration.get_dictionary() choice = params['__choice__'] del params['__choice__'] for param, value in params.items(): param = param.replace(choice, '').replace(':', '') new_params[param] = value if init_params is not None: for param, value in init_params.items(): # These next two lines are different than in the base class - # they allow removing the categorical feature indicator array # in order to not pass it to the no encoding if choice not in param: continue param = param.replace(choice, '').replace(':', '') new_params[param] = value new_params['random_state'] = self.random_state self.new_params = new_params self.choice = self.get_components()[choice](**new_params) return self
def test_uniformfloat_transform(self): """This checks whether a value sampled through the configuration space (it does not happend when the variable is sampled alone) stays equal when it is serialized via JSON and the deserialized again.""" cs = ConfigurationSpace() a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10)) b = cs.add_hyperparameter(NormalFloatHyperparameter('b', 1, 2, log=True)) for i in range(100): config = cs.sample_configuration() value = OrderedDict(sorted(config.get_dictionary().items())) string = json.dumps(value) saved_value = json.loads(string) saved_value = OrderedDict(sorted(byteify(saved_value).items())) self.assertEqual(repr(value), repr(saved_value)) # Next, test whether the truncation also works when initializing the # Configuration with a dictionary for i in range(100): rs = np.random.RandomState(1) value_a = a.sample(rs) value_b = b.sample(rs) values_dict = {'a': value_a, 'b': value_b} config = Configuration(cs, values=values_dict) string = json.dumps(config.get_dictionary()) saved_value = json.loads(string) saved_value = byteify(saved_value) self.assertEqual(values_dict, saved_value)
def test_uniformfloat_transform(self): """This checks whether a value sampled through the configuration space (it does not happend when the variable is sampled alone) stays equal when it is serialized via JSON and the deserialized again.""" cs = ConfigurationSpace() a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10)) b = cs.add_hyperparameter( NormalFloatHyperparameter('b', 1, 2, log=True)) for i in range(100): config = cs.sample_configuration() value = OrderedDict(sorted(config.get_dictionary().items())) string = json.dumps(value) saved_value = json.loads(string) saved_value = OrderedDict(sorted(byteify(saved_value).items())) self.assertEqual(repr(value), repr(saved_value)) # Next, test whether the truncation also works when initializing the # Configuration with a dictionary for i in range(100): rs = np.random.RandomState(1) value_a = a.sample(rs) value_b = b.sample(rs) values_dict = {'a': value_a, 'b': value_b} config = Configuration(cs, values=values_dict) string = json.dumps(config.get_dictionary()) saved_value = json.loads(string) saved_value = byteify(saved_value) self.assertEqual(values_dict, saved_value)
def insert_runhistory(self, config: Configuration, cost: float, time: float, status: StatusType, instance_id: str = "", seed: int = 0, additional_info: dict = frozendict(), origin: DataOrigin = DataOrigin.INTERNAL): config_id = get_id_of_config(config) run_id = self.get_run_id(instance_id, config_id) if instance_id is None: instance_id = "" try: self.Model( run_id=run_id, config_id=config_id, config=config.get_dictionary(), config_origin=config.origin, config_bin=pickle.dumps(config), cost=cost, time=time, instance_id=instance_id, seed=seed, status=status.value, additional_info=dict(additional_info), origin=origin.value, ).save() except Exception as e: pass self.timestamp = datetime.datetime.now()
def set_child_hyperparameters(self, children: List[Tuple[str, EstimatorComponent]], configuration: Dict = None, init_params=None): for node_idx, (node_name, node) in enumerate(children): sub_configuration_space = node.get_hyperparameter_search_space() sub_config_dict = {} for param in configuration: if param.startswith(f'{node_name}:'): value = configuration[param] new_name = param.replace(f'{node_name}:', '', 1) sub_config_dict[new_name] = value sub_configuration = Configuration(sub_configuration_space, values=sub_config_dict) if init_params is not None: sub_init_params_dict = {} for param in init_params: if param.startswith(f'{node_name}:'): value = init_params[param] new_name = param.replace(f'{node_name}:', '', 1) sub_init_params_dict[new_name] = value else: sub_init_params_dict = None if isinstance(node, (ComponentChoice, EstimatorComponent)): node.set_hyperparameters(configuration=sub_configuration.get_dictionary(), init_params=sub_init_params_dict) else: raise NotImplementedError('Not supported yet!')
def process_config_info_pair(self, config: Configuration, info_dict: dict, budget): self.budget2obvs[budget]["locks"].append(config.get_array().copy()) info_dict = deepcopy(info_dict) if config.origin is None: config.origin = "unknown" info_dict.update({"origin": config.origin}) return config.get_dictionary(), info_dict
def remove_resource( self, config_ext: CS.Configuration, as_dict: bool=False) -> Union[CS.Configuration, dict]: x_dct = copy.copy(config_ext.get_dictionary()) del x_dct[self.resource_attr_name] if as_dict: return x_dct else: return CS.Configuration(self.hp_ranges.config_space, values=x_dct)
def get(self, config: CS.Configuration, resource: int) -> CS.Configuration: """ Create extended config with resource added. :param config: :param resource: :return: Extended config """ values = copy.deepcopy(config.get_dictionary()) values[self.resource_attr_name] = resource return CS.Configuration(self.hp_ranges_ext.config_space, values=values)
def test_remove_inactive_parameter(): configuration_space = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("hp1", choices=[0, 1]) hp2 = CategoricalHyperparameter("hp2", choices=['a']) hp3 = UniformIntegerHyperparameter("hp3", lower=0, upper=5, default_value=5) configuration_space.add_hyperparameters([hp1, hp2, hp3]) # If hp1 = 0, then don't allow hp2 not_condition = NotEqualsCondition(hp2, hp1, 0) configuration_space.add_condition(not_condition) allowed_cfg = Configuration(configuration_space, { 'hp1': 1, 'hp2': 'a', 'hp3': 5 }) not_allowed = {'hp1': 0, 'hp2': 'a', 'hp3': 5} with pytest.raises(ValueError): Configuration(configuration_space, not_allowed) # No inactive hp - case: config is CS.configuration transformed = AbstractBenchmark._check_and_cast_configuration( allowed_cfg, configuration_space) assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5} # No inactive hp - case: config is dict transformed = AbstractBenchmark._check_and_cast_configuration( allowed_cfg.get_dictionary(), configuration_space) assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5} # Remove inactive: - case: config is CS.configuration not_allowed_cs = Configuration(configuration_space, { 'hp1': 0, 'hp2': 'a', 'hp3': 5 }, allow_inactive_with_values=True) transformed = AbstractBenchmark._check_and_cast_configuration( not_allowed_cs, configuration_space) assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5} # Remove inactive: - case: config is dict transformed = AbstractBenchmark._check_and_cast_configuration( not_allowed, configuration_space) assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
def split(self, config_ext: CS.Configuration, as_dict: bool=False) -> \ (Union[CS.Configuration, dict], int): """ Split extended config into normal config and resource value. :param config_ext: Extended config :param as_dict: Return config as dict? :return: (config, resource_value) """ x_res = copy.copy(config_ext.get_dictionary()) resource_value = int(x_res[self.resource_attr_name]) del x_res[self.resource_attr_name] if not as_dict: x_res = CS.Configuration(self.hp_ranges.config_space, values=x_res) return x_res, resource_value
def remove_resource( self, config_ext: CS.Configuration, as_dict: bool = False) -> Union[CS.Configuration, dict]: """ Strips away resource attribute and returns normal config :param config_ext: Extended config :param as_dict: Return as dict? :return: config_ext without resource attribute """ x_dct = copy.copy(config_ext.get_dictionary()) del x_dct[self.resource_attr_name] if as_dict: return x_dct else: return CS.Configuration(self.hp_ranges.config_space, values=x_dct)
def transform_dataset(self, ds: Dataset, cid: CandidateId, component: EstimatorComponent, config: Configuration) -> Tuple[np.ndarray, Optional[float]]: component.set_hyperparameters(config.get_dictionary()) if is_classifier(component): score, y_pred, y_prob, models = self._score(ds, component) # TODO crude fix for holdout score. Fix this if y_pred.shape != ds.y.shape: y_pred = models[0].predict(ds.X) y_prob = models[0].predict_proba(ds.X) X = np.hstack((ds.X, y_prob, np.reshape(y_pred, (-1, 1)))) else: models = [component.fit(ds.X, ds.y)] X = models[0].transform(ds.X) score = [None, None] self._store_models(cid, models) return X, score
def get_random_neighborhood(configuration: Configuration, num: int, seed: int) -> List[Configuration]: configuration_space = configuration.configuration_space conf_dict_data = configuration.get_dictionary() array_data = configuration.get_array() neighbor_dict = dict() for key, value in conf_dict_data.items(): neighbor_dict[key] = [ array_data[configuration_space._hyperparameter_idx[key]] ] for hp in configuration.configuration_space.get_hyperparameters(): # trans_data = hp._inverse_transform(conf_dict_data[hp.name]) # neighbors = hp.get_neighbors(trans_data, np.random.RandomState(seed), num, False) # neighbor_dict[hp.name].extend(neighbors) if hp.name not in conf_dict_data: continue neighbors = get_hp_neighbors(hp, conf_dict_data, num, transform=False, seed=seed) neighbor_dict[hp.name].extend(neighbors) neighborhood = [] conf_num = 0 cnt = 0 while conf_num < num and cnt < 5 * num: cnt += 1 data = array_data.copy() # TODO: one exchange neighborhood for key in conf_dict_data.keys(): data[configuration_space._hyperparameter_idx[key]] = random.choice( neighbor_dict[key]) # data[configuration_space._hyperparameter_idx[key]] = sample_hp(neighbor_dict[key], seed) try: config = Configuration(configuration_space, vector=data) config.is_valid_configuration() except Exception as e: pass if config not in neighborhood: neighborhood.append(config) conf_num += 1 assert (len(neighborhood) >= 1) return neighborhood
def remap_resource(self, config_ext: CS.Configuration, resource: int, as_dict: bool = False) -> Union[CS.Configuration, dict]: """ Re-assigns resource value for extended config. :param config_ext: Extended config :param resource: New resource value :param as_dict: Return as dict? :return: """ x_dct = copy.copy(config_ext.get_dictionary()) x_dct[self.resource_attr_name] = resource if as_dict: return x_dct else: return CS.Configuration(self.hp_ranges_ext.config_space, values=x_dct)
def cleanup_pending(self, config: CS.Configuration): """ Removes all pending candidates whose configuration (i.e., lacking the resource attribute) is equal to config. This should be called after an evaluation terminates. For various reasons (e.g., termination due to convergence), pending candidates for this evaluation may still be present. It is also called for a failed evaluation. :param config: See above """ config_dct = config.get_dictionary() def filter_pred(x: PendingEvaluation) -> bool: x_dct = self.configspace_ext.remove_resource( x.candidate, as_dict=True) return x_dct != config_dct self.state_transformer.filter_pending_evaluations(filter_pred)
for uhp in unconditional_hyperparameters: children = configuration_space._children_of[uhp] if len(children) > 0: hyperparameters_with_children.append(uhp) hps.extendleft(hyperparameters_with_children) inactive = set() while len(hps) > 0: hp = hps.pop() children = configuration_space._children_of[hp] for child in children: conditions = configuration_space._parent_conditions_of[child.name] for condition in conditions: if not condition.evaluate_vector(configuration.get_array()): dic = configuration.get_dictionary() try: del dic[child.name] except KeyError: continue configuration = Configuration( configuration_space=configuration_space, values=dic, allow_inactive_with_values=True) inactive.add(child.name) hps.appendleft(child.name) for hp in hyperparameters: if hp.name in inactive: dic = configuration.get_dictionary() try:
def get_resource(self, config_ext: CS.Configuration) -> int: """ :param config_ext: Extended config :return: Value of resource attribute """ return int(config_ext.get_dictionary()[self.resource_attr_name])
def get_random_neighbor(configuration: Configuration, seed: int) -> Configuration: """Draw a random neighbor by changing one parameter of a configuration. * If the parameter is categorical, it changes it to another value. * If the parameter is ordinal, it changes it to the next higher or lower value. * If parameter is a float, draw a random sample If changing a parameter activates new parameters or deactivates previously active parameters, the configuration will be rejected. If more than 10000 configurations were rejected, this function raises a ValueError. Parameters ---------- configuration : Configuration seed : int Used to generate a random state. Returns ------- Configuration The new neighbor. """ random = np.random.RandomState(seed) rejected = True values = copy.deepcopy(configuration.get_dictionary()) while rejected: # First, choose an active hyperparameter active = False iteration = 0 while not active: iteration += 1 if configuration._num_hyperparameters > 1: rand_idx = random.randint(0, configuration._num_hyperparameters - 1) else: rand_idx = 0 value = configuration.get_array()[rand_idx] if np.isfinite(value): active = True hp_name = configuration.configuration_space \ .get_hyperparameter_by_idx(rand_idx) hp = configuration.configuration_space.get_hyperparameter(hp_name) # Only choose if there is a possibility of finding a neigboor if not hp.has_neighbors(): active = False if iteration > 10000: raise ValueError('Probably caught in an infinite loop.') # Get a neighboor and adapt the rest of the configuration if necessary neighbor = hp.get_neighbors(value, random, number=1, transform=True)[0] previous_value = values[hp.name] values[hp.name] = neighbor try: new_configuration = Configuration( configuration.configuration_space, values=values) rejected = False except ValueError as e: values[hp.name] = previous_value return new_configuration
def filter_pred(x: CS.Configuration) -> bool: x_dct = x.get_dictionary() return (x_dct[self.name_last_pos] == self.value_for_last_pos)
def _transform_config(self, config: CS.Configuration) -> CS.Configuration: values = config.get_dictionary() # No copy is done here values[self.name_last_pos] = self.value_for_last_pos return CS.Configuration(self.config_space, values=values)