def __init__(self,
objective_function,
config_space,
time_limit_per_trial=180,
max_runs=200,
model_type='gp_mcmc',
logging_dir='./logs',
initial_configurations=None,
initial_runs=3,
task_id=None,
rng=None):
super().__init__(config_space, task_id, output_dir=logging_dir)
self.logger = super()._get_logger(self.__class__.__name__)
if rng is None:
run_id, rng = get_rng()
self.rng = rng
self.seed = rng.randint(MAXINT)
self.init_num = initial_runs
self.max_iterations = max_runs
self.iteration_id = 0
self.sls_max_steps = None
self.n_sls_iterations = 5
self.sls_n_steps_plateau_walk = 10
self.time_limit_per_trial = time_limit_per_trial
self.default_obj_value = MAXINT
self.configurations = list()
self.failed_configurations = list()
self.perfs = list()
# Initialize the basic component in BO.
self.config_space.seed(rng.randint(MAXINT))
self.objective_function = objective_function
self.model = build_model(model_type=model_type,
config_space=config_space,
rng=self.rng)
self.acquisition_function = EI(self.model)
self.optimizer = InterleavedLocalAndRandomSearch(
acquisition_function=self.acquisition_function,
config_space=self.config_space,
rng=np.random.RandomState(seed=self.seed),
max_steps=self.sls_max_steps,
n_steps_plateau_walk=self.sls_n_steps_plateau_walk,
n_sls_iterations=self.n_sls_iterations)
self._random_search = RandomSearch(self.acquisition_function,
self.config_space, rng)
self.random_configuration_chooser = ChooserProb(prob=0.25, rng=rng)
def __init__(self,
target_hpo_data: Dict,
config_space: ConfigurationSpace,
surrogate_model: BaseFacade,
acq_func: str = 'ei',
source_hpo_data=None,
enable_init_design=False,
num_src_hpo_trial=50,
surrogate_type='rf',
max_runs=200,
logging_dir='./logs',
initial_runs=3,
task_id=None,
random_seed=None):
super().__init__(config_space, task_id, output_dir=logging_dir)
self.logger = super()._get_logger(self.__class__.__name__)
if random_seed is None:
_, rng = get_rng()
random_seed = rng.randint(MAXINT)
self.random_seed = random_seed
self.rng = np.random.RandomState(self.random_seed)
self.source_hpo_data = source_hpo_data
self.num_src_hpo_trial = num_src_hpo_trial
self.surrogate_type = surrogate_type
self.acq_func = acq_func
self.max_iterations = max_runs
self.iteration_id = 0
self.default_obj_value = MAXINT
self.target_hpo_measurements = target_hpo_data
self.configuration_list = list(self.target_hpo_measurements.keys())
print('Target problem space: %d configurations' %
len(self.configuration_list))
self.configurations = list()
self.failed_configurations = list()
self.perfs = list()
self.test_perfs = list()
if enable_init_design:
self.initial_configurations = self.initial_design(initial_runs)
else:
self.initial_configurations = None
if self.initial_configurations is None:
self.init_num = initial_runs
else:
self.init_num = len(self.initial_configurations)
# Initialize the basic component in BO.
self.config_space.seed(self.random_seed)
np.random.seed(self.random_seed)
self.model = surrogate_model
assert self.model.method_id == 'rgpe'
self.acquisition_function = EI(self.model)
self.space_classifier = None
self.build_classifier()
self.random_configuration_chooser = ChooserProb(
prob=0.1, rng=np.random.RandomState(self.random_seed))
# Set the parameter in metric. TODO: add test perf
ys = np.array(list(self.target_hpo_measurements.values()))
if ys.ndim == 2:
assert ys.shape[1] == 2
val_ys = ys[:, 0]
else:
val_ys = ys
self.ys = val_ys
self.y_max, self.y_min = np.max(self.ys), np.min(self.ys)
def create_gp_model(model_type, config_space, types, bounds, rng):
"""
Construct the Gaussian process model that is capable of dealing with categorical hyperparameters.
"""
if rng is None:
_, rng = get_rng(rng)
cov_amp = ConstantKernel(
2.0,
constant_value_bounds=(np.exp(-10), np.exp(2)),
prior=LognormalPrior(mean=0.0, sigma=1.0, rng=rng),
)
cont_dims = np.nonzero(types == 0)[0]
cat_dims = np.nonzero(types != 0)[0]
if len(cont_dims) > 0:
exp_kernel = Matern(
np.ones([len(cont_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cont_dims))],
nu=2.5,
operate_on=cont_dims,
)
if len(cat_dims) > 0:
ham_kernel = HammingKernel(
np.ones([len(cat_dims)]),
[(np.exp(-6.754111155189306), np.exp(0.0858637988771976)) for _ in range(len(cat_dims))],
operate_on=cat_dims,
)
noise_kernel = WhiteKernel(
noise_level=1e-8,
noise_level_bounds=(np.exp(-25), np.exp(2)),
prior=HorseshoePrior(scale=0.1, rng=rng),
)
if len(cont_dims) > 0 and len(cat_dims) > 0:
# both
kernel = cov_amp * (exp_kernel * ham_kernel) + noise_kernel
elif len(cont_dims) > 0 and len(cat_dims) == 0:
# only cont
kernel = cov_amp * exp_kernel + noise_kernel
elif len(cont_dims) == 0 and len(cat_dims) > 0:
# only cont
kernel = cov_amp * ham_kernel + noise_kernel
else:
raise ValueError()
# seed = rng.randint(0, 2 ** 20)
if model_type == 'gp_mcmc':
n_mcmc_walkers = 3 * len(kernel.theta)
if n_mcmc_walkers % 2 == 1:
n_mcmc_walkers += 1
model = GaussianProcessMCMC(
config_space,
types=types,
bounds=bounds,
kernel=kernel,
n_mcmc_walkers=n_mcmc_walkers,
chain_length=250,
burnin_steps=250,
normalize_y=True,
seed=rng.randint(low=0, high=10000),
)
elif model_type == 'gp':
model = GaussianProcess(
config_space,
types=types,
bounds=bounds,
kernel=kernel,
normalize_y=True,
seed=rng.randint(low=0, high=10000),
)
else:
raise ValueError("Invalid model str %s!" % model_type)
return model
def __init__(self,
target_hpo_data: Dict,
config_space: ConfigurationSpace,
surrogate_model: BaseFacade,
acq_func: str = 'ei',
mode='best',
model='gp',
source_hpo_data=None,
enable_init_design=False,
num_src_hpo_trial=50,
surrogate_type='rf',
max_runs=200,
logging_dir='./logs',
initial_runs=3,
task_id=None,
random_seed=None):
super().__init__(config_space, task_id, output_dir=logging_dir)
self.logger = super()._get_logger(self.__class__.__name__)
if random_seed is None:
_, rng = get_rng()
random_seed = rng.randint(MAXINT)
self.random_seed = random_seed
self.rng = np.random.RandomState(self.random_seed)
self.source_hpo_data = source_hpo_data
self.num_src_hpo_trial = num_src_hpo_trial
self.surrogate_type = surrogate_type
self.acq_func = acq_func
self.mode = mode
self.clf_type = model
print(mode, model)
self.max_iterations = max_runs
self.iteration_id = 0
self.default_obj_value = MAXINT
self.target_hpo_measurements = target_hpo_data
self.configuration_list = list(self.target_hpo_measurements.keys())
print('Target problem space: %d configurations' %
len(self.configuration_list))
self.configurations = list()
self.failed_configurations = list()
self.perfs = list()
self.test_perfs = list()
self.reduce_cnt = 0
if enable_init_design:
self.initial_configurations = self.initial_design(initial_runs)
else:
self.initial_configurations = None
if self.initial_configurations is None:
self.init_num = initial_runs
else:
self.init_num = len(self.initial_configurations)
# Initialize the basic component in BO.
self.config_space.seed(self.random_seed)
np.random.seed(self.random_seed)
self.model = surrogate_model
self.acquisition_function = EI(self.model)
self.space_classifier = None
self.random_configuration_chooser = ChooserProb(
prob=0.1, rng=np.random.RandomState(self.random_seed))
# self.prop_eta = 0.9
# self.prop_init = 0.3
# Set the parameter in metric. TODO: add test perf
ys = np.array(list(self.target_hpo_measurements.values()))
if ys.ndim == 2:
assert ys.shape[1] == 2
val_ys = ys[:, 0]
else:
val_ys = ys
self.ys = val_ys
self.y_max, self.y_min = np.max(self.ys), np.min(self.ys)
self.reduce_cnt = 0
self.p_min = 5
self.p_max = 50
self.use_correct_rate = False
if self.mode in ['box', 'ellipsoid']:
continuous_types = (UniformFloatHyperparameter,
UniformIntegerHyperparameter)
self.continuous_mask = [
isinstance(hp, continuous_types)
for hp in self.config_space.get_hyperparameters()
]
self.calculate_box_area()
self.space_threshold = 0.04