def __init__(self,
config_space: ConfigurationSpace,
source_hpo_data: List,
seed: int,
history_dataset_features: List = None,
num_src_hpo_trial: int = 50,
surrogate_type='rf'):
self.method_id = None
self.config_space = config_space
self.random_seed = seed
self.num_src_hpo_trial = num_src_hpo_trial
self.source_hpo_data = source_hpo_data
self.source_surrogates = None
self.target_surrogate = None
self.history_dataset_features = history_dataset_features
# The number of source problems.
if source_hpo_data is not None:
self.K = len(source_hpo_data)
if history_dataset_features is not None:
assert len(history_dataset_features) == self.K
self.surrogate_type = surrogate_type
self.types, self.bounds = get_types(config_space)
self.instance_features = None
self.var_threshold = VERY_SMALL_NUMBER
self.w = None
self.eta_list = list()
# meta features.
self.meta_feature_scaler = None
self.meta_feature_imputer = None
self.target_weight = list()
self.logger = get_logger(self.__class__.__name__)
def evaluate(mth, run_i, seed):
print(mth, run_i, seed, '===== start =====', flush=True)
def objective_function(config):
y = problem.evaluate_config(config)
return y
from cma import CMAEvolutionStrategy
from litebo.utils.util_funcs import get_types
from litebo.utils.config_space import Configuration
types, bounds = get_types(cs)
assert all(types == 0)
# Check Constant Hyperparameter
const_idx = list()
for i, bound in enumerate(bounds):
if np.isnan(bound[1]):
const_idx.append(i)
hp_num = len(bounds) - len(const_idx)
es = CMAEvolutionStrategy(hp_num * [0], 0.99, inopts={'bounds': [0, 1], 'seed': seed})
global_start_time = time.time()
global_trial_counter = 0
config_list = []
perf_list = []
time_list = []
eval_num = 0
while eval_num < max_runs:
X = es.ask(number=es.popsize)
_X = X.copy()
for i in range(len(_X)):
for index in const_idx:
_X[i] = np.insert(_X[i], index, 0) # np.insert returns a copy
# _X = np.asarray(_X)
values = []
for xi in _X:
# convert array to Configuration
config = Configuration(cs, vector=xi)
perf = objective_function(config)
global_time = time.time() - global_start_time
global_trial_counter += 1
values.append(perf)
print('=== CMAES Trial %d: %s perf=%f global_time=%f' % (global_trial_counter, config, perf, global_time))
config_list.append(config)
perf_list.append(perf)
time_list.append(global_time)
values = np.reshape(values, (-1,))
es.tell(X, values)
eval_num += es.popsize
print('===== Total evaluation times=%d. Truncate to max_runs=%d.' % (eval_num, max_runs))
config_list = config_list[:max_runs]
perf_list = perf_list[:max_runs]
time_list = time_list[:max_runs]
return config_list, perf_list, time_list
def build_surrogate(func_str='prf',
config_space=None,
rng=None,
history_hpo_data=None):
assert config_space is not None
func_str = func_str.lower()
types, bounds = get_types(config_space)
seed = rng.randint(MAXINT)
if func_str == 'prf':
return RandomForestWithInstances(types=types, bounds=bounds, seed=seed)
elif func_str.startswith('gp'):
return create_gp_model(model_type=func_str,
config_space=config_space,
types=types,
bounds=bounds,
rng=rng)
elif func_str.startswith('mfgpe'):
from litebo.surrogate.tlbo.mfgpe import MFGPE
inner_surrogate_type = 'prf'
return MFGPE(config_space,
history_hpo_data,
seed,
surrogate_type=inner_surrogate_type,
num_src_hpo_trial=-1)
elif func_str.startswith('tlbo'):
print('the current surrogate is', func_str)
if 'rgpe' in func_str:
from litebo.surrogate.tlbo.rgpe import RGPE
inner_surrogate_type = func_str.split('_')[-1]
return RGPE(config_space,
history_hpo_data,
seed,
surrogate_type=inner_surrogate_type,
num_src_hpo_trial=-1)
elif 'sgpr' in func_str:
from litebo.surrogate.tlbo.stacking_gpr import SGPR
inner_surrogate_type = func_str.split('_')[-1]
return SGPR(config_space,
history_hpo_data,
seed,
surrogate_type=inner_surrogate_type,
num_src_hpo_trial=-1)
elif 'topov3' in func_str:
from litebo.surrogate.tlbo.topo_variant3 import TOPO_V3
inner_surrogate_type = func_str.split('_')[-1]
return TOPO_V3(config_space,
history_hpo_data,
seed,
surrogate_type=inner_surrogate_type,
num_src_hpo_trial=-1)
else:
raise ValueError('Invalid string %s for tlbo surrogate!' %
func_str)
else:
raise ValueError('Invalid string %s for surrogate!' % func_str)
def __init__(self,
objective_function,
config_space,
sample_strategy='bo',
time_limit_per_trial=180,
max_runs=200,
logging_dir='logs',
initial_configurations=None,
initial_batch=1,
batch_size=3,
task_id=None,
rng=None):
super().__init__(config_space, task_id, output_dir=logging_dir)
self.logger = super()._get_logger(self.__class__.__name__)
if rng is None:
run_id, rng = get_rng()
self.batch_size = batch_size
self.init_batch = initial_batch
self.max_iterations = max_runs
self.iteration_id = 0
self.sls_max_steps = None
self.n_sls_iterations = 5
self.sls_n_steps_plateau_walk = 10
self.time_limit_per_trial = time_limit_per_trial
self.default_obj_value = MAXINT
self.sample_strategy = sample_strategy
self.configurations = list()
self.failed_configurations = list()
self.perfs = list()
# Initialize the basic component in BO.
self.config_space.seed(rng.randint(MAXINT))
self.objective_function = objective_function
types, bounds = get_types(config_space)
# TODO: what is the feature array.
self.model = RandomForestWithInstances(types=types,
bounds=bounds,
seed=rng.randint(MAXINT))
if self.sample_strategy == 'local_penalization':
self.acquisition_function = LPEI(self.model)
else:
self.acquisition_function = EI(self.model)
self.optimizer = InterleavedLocalAndRandomSearch(
acquisition_function=self.acquisition_function,
config_space=self.config_space,
rng=np.random.RandomState(seed=rng.randint(MAXINT)),
max_steps=self.sls_max_steps,
n_steps_plateau_walk=self.sls_n_steps_plateau_walk,
n_sls_iterations=self.n_sls_iterations)
self._random_search = RandomSearch(self.acquisition_function,
self.config_space, rng)
self.random_configuration_chooser = ChooserProb(prob=0.25, rng=rng)
def __init__(
self,
acquisition_function: AbstractAcquisitionFunction,
config_space: ConfigurationSpace,
rand_prob: float = 0.0,
rng: Union[bool, np.random.RandomState] = None,
):
super().__init__(acquisition_function, config_space, rng)
self.random_chooser = ChooserProb(prob=rand_prob, rng=rng)
types, bounds = get_types(self.config_space)
assert all(types == 0)
self.bounds = bounds
def __init__(self,
acquisition_function: AbstractAcquisitionFunction,
config_space: ConfigurationSpace,
rng: Union[bool, np.random.RandomState] = None,
batch_size=None,
rand_prob=0.0):
super().__init__(acquisition_function, config_space, rng)
self.random_chooser = ChooserProb(prob=rand_prob, rng=rng)
if batch_size is None:
types, bounds = get_types(self.config_space)
dim = np.sum(types == 0)
self.batch_size = min(5000, max(2000, 200 * dim))
else:
self.batch_size = batch_size
def maximize(self, runhistory: HistoryContainer, num_points: int,
**kwargs) -> Iterable[Tuple[float, Configuration]]:
try:
from cma import CMAEvolutionStrategy
except ImportError:
raise ImportError("Package cma is not installed!")
types, bounds = get_types(self.config_space)
assert all(types == 0)
# Check Constant Hyperparameter
const_idx = list()
for i, bound in enumerate(bounds):
if np.isnan(bound[1]):
const_idx.append(i)
hp_num = len(bounds) - len(const_idx)
es = CMAEvolutionStrategy(hp_num * [0],
0.99,
inopts={'bounds': [0, 1]})
eval_num = 0
next_configs_by_acq_value = list()
while eval_num < num_points:
X = es.ask(number=es.popsize)
_X = X.copy()
for i in range(len(_X)):
for index in const_idx:
_X[i] = np.insert(_X[i], index, 0)
_X = np.asarray(_X)
values = self.acquisition_function._compute(_X)
values = np.reshape(values, (-1, ))
es.tell(X, values)
next_configs_by_acq_value.extend([(values[i], _X[i])
for i in range(es.popsize)])
eval_num += es.popsize
next_configs_by_acq_value.sort(reverse=True, key=lambda x: x[0])
next_configs_by_acq_value = [_[1] for _ in next_configs_by_acq_value]
next_configs_by_acq_value = [
Configuration(self.config_space, vector=array)
for array in next_configs_by_acq_value
]
challengers = ChallengerList(next_configs_by_acq_value,
self.config_space, self.random_chooser)
self.random_chooser.next_smbo_iteration()
return challengers
def __init__(
self,
acquisition_function: AbstractAcquisitionFunction,
config_space: ConfigurationSpace,
rand_prob: float = 0.0,
rng: Union[bool, np.random.RandomState] = None,
):
super().__init__(acquisition_function, config_space, rng)
self.random_chooser = ChooserProb(prob=rand_prob, rng=rng)
types, bounds = get_types(self.config_space)
assert all(types == 0)
self.bounds = bounds
options = dict(disp=False, maxiter=1000)
self.scipy_config = dict(tol=None, method='L-BFGS-B', options=options)
def setup_bo_basics(self):
if self.num_objs == 1 or self.acq_type == 'parego':
self.surrogate_model = build_surrogate(
func_str=self.surrogate_type,
config_space=self.config_space,
rng=self.rng,
history_hpo_data=self.history_bo_data)
else: # multi-objectives
self.surrogate_model = [
build_surrogate(func_str=self.surrogate_type,
config_space=self.config_space,
rng=self.rng,
history_hpo_data=self.history_bo_data)
for _ in range(self.num_objs)
]
if self.num_constraints > 0:
self.constraint_models = [
build_surrogate(func_str=self.constraint_surrogate_type,
config_space=self.config_space,
rng=self.rng)
for _ in range(self.num_constraints)
]
if self.acq_type in ['mesmo', 'mesmoc', 'mesmoc2', 'usemo']:
types, bounds = get_types(self.config_space)
self.acquisition_function = build_acq_func(
func_str=self.acq_type,
model=self.surrogate_model,
constraint_models=self.constraint_models,
types=types,
bounds=bounds)
else:
self.acquisition_function = build_acq_func(
func_str=self.acq_type,
model=self.surrogate_model,
constraint_models=self.constraint_models,
ref_point=self.ref_point)
if self.acq_type == 'usemo':
self.acq_optimizer_type = 'usemo_optimizer'
elif self.acq_type.startswith('mesmo'):
self.acq_optimizer_type = 'mesmo_optimizer'
self.optimizer = build_optimizer(func_str=self.acq_optimizer_type,
acq_func=self.acquisition_function,
config_space=self.config_space,
rng=self.rng)
def setup_bo_basics(self):
if self.num_objs == 1:
self.surrogate_model = build_surrogate(
func_str=self.surrogate_type,
config_space=self.config_space,
rng=self.rng,
history_hpo_data=self.history_bo_data)
else: # multi-objectives
self.surrogate_model = [
build_surrogate(func_str=self.surrogate_type,
config_space=self.config_space,
rng=self.rng,
history_hpo_data=self.history_bo_data)
for _ in range(self.num_objs)
]
if self.num_constraints > 0:
self.constraint_models = [
build_surrogate(func_str=self.constraint_surrogate_type,
config_space=self.config_space,
rng=self.rng)
for _ in range(self.num_constraints)
]
self.acquisition_function = build_acq_func(
func_str=self.acq_type,
model=self.surrogate_model,
constraint_models=self.constraint_models,
mc_times=self.mc_times,
ref_point=self.ref_point)
self.optimizer = build_optimizer(func_str=self.acq_optimizer_type,
acq_func=self.acquisition_function,
config_space=self.config_space,
rng=self.rng)
if self.use_trust_region:
types, bounds = get_types(self.config_space)
cont_dim = np.sum(types == 0)
self.turbo_state = TurboState(cont_dim)
else:
self.turbo_state = None
def __init__(self,
config_space: ConfigurationSpace,
size,
lower_bounds=None,
upper_bounds=None,
random_state=None):
"""
Parameters
----------
config_space : ConfigurationSpace
ConfigurationSpace to do sampling.
size : int N
Number of samples.
lower_bounds : lower bounds in [0, 1] for continuous dimensions (optional)
upper_bounds : upper bounds in [0, 1] for continuous dimensions (optional)
"""
self.config_space = config_space
types, bounds = get_types(config_space)
self.search_dims = []
for i in range(len(types)):
if types[i] == 0 and bounds[i][1] == 1.0: # Integer and float
self.search_dims.append((0.0, 1.0))
elif types[i] > 0: # Categorical
self.search_dims.append(list(range(types[i])))
else:
raise NotImplementedError()
self.size = size
default_lb, default_ub = zip(*bounds)
self.lower_bounds = np.array(
default_lb) if lower_bounds is None else np.clip(
lower_bounds, default_lb, default_ub)
self.upper_bounds = np.array(
default_ub) if upper_bounds is None else np.clip(
upper_bounds, default_lb, default_ub)
self.rng = check_random_state(random_state)