def test_get_str_array():
with pytest.raises(AssertionError) as error:
utils_logger.get_str_array(123)
with pytest.raises(AssertionError) as error:
utils_logger.get_str_array(12.3)
str_ = utils_logger.get_str_array(np.array([1, 2, 3]))
print(str_)
assert str_ == '[1, 2, 3]'
str_ = utils_logger.get_str_array(np.array([1.1, 2.5, 3.0]))
print(str_)
assert str_ == '[1.100, 2.500, 3.000]'
str_ = utils_logger.get_str_array(np.array([[1, 2, 3], [2, 2, 2]]))
print(str_)
assert str_ == '[[1, 2, 3],\n[2, 2, 2]]'
str_ = utils_logger.get_str_array(
np.array([[1.1, 2.2, 3.33], [2.2, 2.4, 2.9]]))
print(str_)
assert str_ == '[[1.100, 2.200, 3.330],\n[2.200, 2.400, 2.900]]'
str_ = utils_logger.get_str_array(
np.array([[[1, 2, 3], [2, 2, 2]], [[1, 2, 3], [2, 2, 2]]]))
print(str_)
assert str_ == '[[[1, 2, 3],\n[2, 2, 2]],\n[[1, 2, 3],\n[2, 2, 2]]]'
str_ = utils_logger.get_str_array(
np.array([[[1.1, 2.2, 3.33], [2.2, 2.4, 2.9]],
[[1.1, 2.2, 3.33], [2.2, 2.4, 2.9]]]))
print(str_)
assert str_ == '[[[1.100, 2.200, 3.330],\n[2.200, 2.400, 2.900]],\n[[1.100, 2.200, 3.330],\n[2.200, 2.400, 2.900]]]'
def get_samples(
self,
str_sampling_method: str,
fun_objective: constants.TYPING_UNION_CALLABLE_NONE = None,
num_samples: int = constants.NUM_SAMPLES_AO,
seed: constants.TYPING_UNION_INT_NONE = None,
) -> np.ndarray:
"""
It returns `num_samples` examples, sampled by a sampling method `str_sampling_method`.
:param str_sampling_method: the name of sampling method.
:type str_sampling_method: str.
:param fun_objective: None, or objective function.
:type fun_objective: NoneType or callable, optional
:param num_samples: the number of samples.
:type num_samples: int., optional
:param seed: None, or random seed.
:type seed: NoneType or int., optional
:returns: sampled examples. Shape: (`num_samples`, d).
:rtype: numpy.ndarray
:raises: AssertionError
"""
assert isinstance(str_sampling_method, str)
assert callable(fun_objective) or fun_objective is None
assert isinstance(num_samples, int)
assert isinstance(seed, (int, constants.TYPE_NONE))
assert str_sampling_method in constants.ALLOWED_SAMPLING_METHOD
if str_sampling_method == 'grid':
assert fun_objective is not None
if self.debug:
self.logger.debug(
'For this option, num_samples is used as num_grids.')
samples = self._get_samples_grid(num_grids=num_samples)
samples = utils_bo.get_best_acquisition_by_evaluation(
samples, fun_objective)
elif str_sampling_method == 'uniform':
samples = self._get_samples_uniform(num_samples, seed=seed)
elif str_sampling_method == 'gaussian':
samples = self._get_samples_gaussian(num_samples, seed=seed)
elif str_sampling_method == 'sobol':
samples = self._get_samples_sobol(num_samples, seed=seed)
elif str_sampling_method == 'halton':
samples = self._get_samples_halton(num_samples, seed=seed)
else:
raise NotImplementedError(
'get_samples: allowed str_sampling_method,\
but it is not implemented.')
if self.debug:
self.logger.debug('samples:\n%s',
utils_logger.get_str_array(samples))
return samples
def get_initial(
self,
str_initial_method,
fun_objective=None,
int_samples=constants.NUM_ACQ_SAMPLES,
int_seed=None,
):
"""
It returns a single example or `int_samples` examples, sampled by a certian method `str_initial_method`.
:param str_initial_method: the name of sampling method.
:type str_initial_method: str.
:param fun_objective: None, or objective function.
:type fun_objective: NoneType or function, optional
:param int_samples: the number of samples.
:type int_samples: int., optional
:param int_seed: None, or random seed.
:type int_seed: NoneType or int., optional
:returns: sampled examples. Shape: (1, d) or (`int_samples`, d).
:rtype: numpy.ndarray
:raises: AssertionError
"""
assert isinstance(str_initial_method, str)
assert callable(fun_objective) or fun_objective is None
assert isinstance(int_samples, int)
assert isinstance(int_seed, int) or int_seed is None
if str_initial_method == 'grid':
assert fun_objective is not None
if self.debug:
logger.debug('int_samples is ignored, because grid is chosen.')
arr_initials = self._get_initial_grid()
arr_initials = get_best_acquisition(arr_initials, fun_objective)
elif str_initial_method == 'uniform':
arr_initials = self._get_initial_uniform(int_samples,
int_seed=int_seed)
elif str_initial_method == 'sobol':
arr_initials = self._get_initial_sobol(int_samples,
int_seed=int_seed)
elif str_initial_method == 'latin':
raise NotImplementedError('get_initial: latin')
else:
raise NotImplementedError(
'get_initial: allowed str_initial_method, but it is not implemented.'
)
if self.debug:
logger.debug('arr_initials:\n{}'.format(
utils_logger.get_str_array(arr_initials)))
return arr_initials
def optimize(self,
num_init: int,
seed: constants.TYPING_UNION_INT_NONE=None,
) -> constants.TYPING_TUPLE_FIVE_ARRAYS:
"""
It returns the optimization results and times consumed, given
the number of initial samples `num_init` and a random seed
`seed`.
:param num_init: the number of initial samples.
:type num_init: int.
:param seed: None, or a random seed.
:type seed: NoneType or int., optional
:returns: a tuple of acquired samples, their function values, overall
times consumed per iteration, time consumed in modeling Gaussian process
regression, and time consumed in acquisition function optimization.
Shape: ((`num_init` + `num_iter`, d), (`num_init` + `num_iter`, 1),
(`num_init` + `num_iter`, ), (`num_iter`, ), (`num_iter`, )),
or ((`num_init` + `num_iter`, m, d),
(`num_init` + `num_iter`, m, 1), (`num_init` + `num_iter`, ),
(`num_iter`, ),
(`num_iter`, )),
where d is a dimensionality of the problem we are solving
and m is a cardinality of sets.
:rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
:raises: AssertionError
"""
assert isinstance(num_init, int)
assert isinstance(seed, (int, type(None)))
assert num_init > 0
self.print_info(num_init, seed)
time_start = time.time()
X_init = self.model_bo.get_initials(self.str_initial_method_bo, num_init, seed=seed)
if self.debug:
self.model_bo.logger.debug('X_init:\n%s', utils_logger.get_str_array(X_init))
X, Y, time_all, time_surrogate, time_acq = self.optimize_with_initial_inputs(X_init)
time_end = time.time()
if self.debug:
self.model_bo.logger.debug('overall time consumed including initializations: %.4f sec.',
time_end - time_start)
return X, Y, time_all, time_surrogate, time_acq
def print_info(self, num_init, seed):
"""
It returns the optimization results and times consumed, given
inital inputs `X`.
:param num_init: the number of initial points.
:type num_init: int.
:param seed: a random seed.
:type seed: int.
:returns: None
:rtype: NoneType
"""
self.model_bo.logger.info('====================')
self.model_bo.logger.info('range_X:\n%s', utils_logger.get_str_array(self.range_X))
self.model_bo.logger.info('num_init: %d', num_init)
self.model_bo.logger.info('num_iter: %d', self.num_iter)
self.model_bo.logger.info('str_surrogate: %s', self.str_surrogate)
if self.str_surrogate in constants.ALLOWED_SURROGATE:
self.model_bo.logger.info('str_cov: %s', self.str_cov)
self.model_bo.logger.info('str_acq: %s', self.str_acq)
self.model_bo.logger.info('normalize_Y: %s', self.normalize_Y)
if self.str_surrogate in constants.ALLOWED_SURROGATE:
self.model_bo.logger.info('use_ard: %s', self.use_ard)
self.model_bo.logger.info('str_initial_method_bo: %s', self.str_initial_method_bo)
self.model_bo.logger.info('str_sampling_method_ao: %s', self.str_sampling_method_ao)
if self.str_surrogate in ['gp']:
self.model_bo.logger.info('str_optimizer_method_gp: %s', self.str_optimizer_method_gp)
if self.str_surrogate in ['tp']:
self.model_bo.logger.info('str_optimizer_method_tp: %s', self.str_optimizer_method_tp)
self.model_bo.logger.info('str_optimizer_method_bo: %s', self.str_optimizer_method_bo)
if self.str_surrogate in ['gp']:
self.model_bo.logger.info('str_mlm_method: %s', self.str_mlm_method)
self.model_bo.logger.info('str_modelselection_method: %s',
self.str_modelselection_method)
self.model_bo.logger.info('num_samples_ao: %d', self.num_samples_ao)
self.model_bo.logger.info('seed: %s', seed)
self.model_bo.logger.info('debug: %s', self.debug)
self.model_bo.logger.info('====================')
def _get_next_best_sample(self,
next_sample: np.ndarray,
X: np.ndarray,
next_samples: np.ndarray,
acq_vals: np.ndarray,
) -> np.ndarray:
"""
It returns the next best sample in terms of acquisition function values.
:param next_sample: the next sample acquired.
:type next_sample: np.ndarray
:param X: the samples evaluated so far.
:type X: np.ndarray
:param next_samples: the candidates of the next sample.
:type next_samples: np.ndarray
:param acq_vals: the values of acquisition function over `next_samples`.
:type acq_vals: np.ndarray
:returns: the next best sample. Shape: (d, ).
:rtype: numpy.ndarray
:raises: AssertionError
"""
assert isinstance(next_sample, np.ndarray)
assert isinstance(X, np.ndarray)
assert isinstance(next_samples, np.ndarray)
assert isinstance(acq_vals, np.ndarray)
if np.where(np.linalg.norm(next_sample - X, axis=1)\
< constants.TOLERANCE_DUPLICATED_ACQ)[0].shape[0] > 0: # pragma: no cover
next_sample = utils_bo.get_next_best_acquisition(
next_samples, acq_vals, X)
if self.debug:
self.model_bo.logger.debug('next_sample is repeated, so next best is selected.\
next_sample: %s', utils_logger.get_str_array(next_sample))
return next_sample
def _optimize(self, fun_negative_acquisition: constants.TYPING_CALLABLE,
str_sampling_method: str,
num_samples: int) -> constants.TYPING_TUPLE_TWO_ARRAYS:
"""
It optimizes `fun_negative_function` with `self.str_optimizer_method_bo`.
`num_samples` examples are determined by `str_sampling_method`, to
start acquisition function optimization.
:param fun_objective: negative acquisition function.
:type fun_objective: callable
:param str_sampling_method: the name of sampling method.
:type str_sampling_method: str.
:param num_samples: the number of samples.
:type num_samples: int.
:returns: tuple of next point to evaluate and all candidates
determined by acquisition function optimization.
Shape: ((d, ), (`num_samples`, d)).
:rtype: (numpy.ndarray, numpy.ndarray)
"""
list_next_point = []
if self.str_optimizer_method_bo == 'L-BFGS-B':
list_bounds = self._get_bounds()
initials = self.get_samples(str_sampling_method,
fun_objective=fun_negative_acquisition,
num_samples=num_samples)
for arr_initial in initials:
next_point = minimize(fun_negative_acquisition,
x0=arr_initial,
bounds=list_bounds,
method=self.str_optimizer_method_bo,
options={'disp': False})
next_point_x = next_point.x
list_next_point.append(next_point_x)
if self.debug:
self.logger.debug('acquired sample: %s',
utils_logger.get_str_array(next_point_x))
elif self.str_optimizer_method_bo == 'DIRECT': # pragma: no cover
self.logger.debug('num_samples is ignored.')
list_bounds = self._get_bounds()
next_point = directminimize(
fun_negative_acquisition,
bounds=list_bounds,
maxf=88888,
)
next_point_x = next_point.x
list_next_point.append(next_point_x)
elif self.str_optimizer_method_bo == 'CMA-ES':
self.logger.debug('num_samples is ignored.')
list_bounds = self._get_bounds()
list_bounds = np.array(list_bounds)
def fun_wrapper(f):
def g(bx):
return f(bx)[0]
return g
initials = self.get_samples(str_sampling_method,
fun_objective=fun_negative_acquisition,
num_samples=1)
cur_sigma0 = np.mean(list_bounds[:, 1] - list_bounds[:, 0]) / 4.0
next_point_x = cma.fmin(fun_wrapper(fun_negative_acquisition),
initials[0],
cur_sigma0,
options={
'bounds':
[list_bounds[:, 0], list_bounds[:, 1]],
'verbose':
-1,
'maxfevals':
1e5
})[0]
list_next_point.append(next_point_x)
next_points = np.array(list_next_point)
next_point = utils_bo.get_best_acquisition_by_evaluation(
next_points, fun_negative_acquisition)[0]
return next_point, next_points
def _optimize(self, fun_negative_acquisition, str_initial_method,
int_samples):
"""
It optimizes `fun_negative_function` with `self.str_optimizer_method_bo`.
`int_samples` examples are determined by `str_initial_method`, to start acquisition function optimization.
:param fun_objective: negative acquisition function.
:type fun_objective: function
:param str_initial_method: the name of sampling method.
:type str_initial_method: str.
:param int_samples: the number of samples.
:type int_samples: int.
:returns: tuple of next point to evaluate and all candidates determined by acquisition function optimization. Shape: ((d, ), (`int_samples`, d)).
:rtype: (numpy.ndarray, numpy.ndarray)
"""
list_next_point = []
if self.str_optimizer_method_bo == 'L-BFGS-B':
list_bounds = self._get_bounds()
arr_initials = self.get_initial(
str_initial_method,
fun_objective=fun_negative_acquisition,
int_samples=int_samples)
for arr_initial in arr_initials:
next_point = minimize(fun_negative_acquisition,
x0=arr_initial,
bounds=list_bounds,
method=self.str_optimizer_method_bo,
options={'disp': False})
next_point_x = next_point.x
list_next_point.append(next_point_x)
if self.debug:
logger.debug('acquired sample: {}'.format(
utils_logger.get_str_array(next_point_x)))
elif self.str_optimizer_method_bo == 'DIRECT': # pragma: no cover
list_bounds = self._get_bounds()
next_point = directminimize(
fun_negative_acquisition,
bounds=list_bounds,
maxf=88888,
)
next_point_x = next_point.x
list_next_point.append(next_point_x)
elif self.str_optimizer_method_bo == 'CMA-ES':
list_bounds = self._get_bounds()
list_bounds = np.array(list_bounds)
def fun_wrapper(f):
def g(bx):
return f(bx)[0]
return g
arr_initials = self.get_initial(
str_initial_method,
fun_objective=fun_negative_acquisition,
int_samples=1)
cur_sigma0 = np.mean(list_bounds[:, 1] - list_bounds[:, 0]) / 4.0
next_point_x = cma.fmin(fun_wrapper(fun_negative_acquisition),
arr_initials[0],
cur_sigma0,
options={
'bounds':
[list_bounds[:, 0], list_bounds[:, 1]],
'verbose':
-1,
'maxfevals':
1e5
})[0]
list_next_point.append(next_point_x)
next_points = np.array(list_next_point)
next_point = get_best_acquisition(next_points,
fun_negative_acquisition)
return next_point.flatten(), next_points
def optimize_with_all_initial_information(self,
X: np.ndarray, Y: np.ndarray,
) -> constants.TYPING_TUPLE_FIVE_ARRAYS:
"""
It returns the optimization results and times consumed, given
inital inputs `X` and their corresponding outputs `Y`.
:param X: initial inputs. Shape: (n, d) or (n, m, d).
:type X: numpy.ndarray
:param Y: initial outputs. Shape: (n, 1).
:type Y: numpy.ndarray
:returns: a tuple of acquired samples, their function values, overall
times consumed per iteration, time consumed in modeling Gaussian process
regression, and time consumed in acquisition function optimization.
Shape: ((n + `num_iter`, d), (n + `num_iter`, 1),
(`num_iter`, ), (`num_iter`, ), (`num_iter`, )),
or ((n + `num_iter`, m, d),
(n + `num_iter`, m, 1), (`num_iter`, ), (`num_iter`, ),
(`num_iter`, )).
:rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
:raises: AssertionError
"""
assert isinstance(X, np.ndarray)
assert isinstance(Y, np.ndarray)
assert len(X.shape) == 2
assert len(Y.shape) == 2
assert X.shape[0] == Y.shape[0]
assert Y.shape[1] == 1
time_start = time.time()
X_ = X
Y_ = Y
time_all_ = []
time_surrogate_ = []
time_acq_ = []
pbar = tqdm(range(0, self.num_iter))
for ind_iter in pbar:
self.model_bo.logger.info('Iteration %d', ind_iter + 1)
time_iter_start = time.time()
next_sample, dict_info = self.optimize_single_iteration(X_, Y_)
next_samples = dict_info['next_points']
acq_vals = dict_info['acquisitions']
time_surrogate = dict_info['time_surrogate']
time_acq = dict_info['time_acq']
if self.debug:
self.model_bo.logger.debug('next_sample: %s',
utils_logger.get_str_array(next_sample))
next_sample = self._get_next_best_sample(next_sample, X_, next_samples, acq_vals)
X_ = np.vstack((X_, next_sample))
time_to_evaluate_start = time.time()
Y_ = np.vstack((Y_, self.fun_target(next_sample)))
time_to_evaluate_end = time.time()
if self.debug:
self.model_bo.logger.debug('time consumed to evaluate: %.4f sec.',
time_to_evaluate_end - time_to_evaluate_start)
time_iter_end = time.time()
time_all_.append(time_iter_end - time_iter_start)
time_surrogate_.append(time_surrogate)
time_acq_.append(time_acq)
time_end = time.time()
if self.debug:
self.model_bo.logger.debug('overall time consumed in single BO round: %.4f sec.',
time_end - time_start)
time_all_ = np.array(time_all_)
time_surrogate_ = np.array(time_surrogate_)
time_acq_ = np.array(time_acq_)
return X_, Y_, time_all_, time_surrogate_, time_acq_
def run_single_round(
model_bo: bo.BO,
fun_target: constants.TYPING_CALLABLE,
num_init: int,
num_iter: int,
str_initial_method_bo: str = constants.STR_INITIALIZING_METHOD_BO,
str_sampling_method_ao: str = constants.STR_SAMPLING_METHOD_AO,
num_samples_ao: int = constants.NUM_SAMPLES_AO,
str_mlm_method: str = constants.STR_MLM_METHOD,
seed: constants.TYPING_UNION_INT_NONE = None
) -> constants.TYPING_TUPLE_FIVE_ARRAYS:
"""
It optimizes `fun_target` for `num_iter` iterations with given
`model_bo` and `num_init` initial examples.
Initial examples are sampled by `get_initials` method in `model_bo`.
It returns the optimization results and execution times.
:param model_bo: Bayesian optimization model.
:type model_bo: bayeso.bo.BO
:param fun_target: a target function.
:type fun_target: callable
:param num_init: the number of initial examples for Bayesian optimization.
:type num_init: int.
:param num_iter: the number of iterations for Bayesian optimization.
:type num_iter: int.
:param str_initial_method_bo: the name of initialization method for
sampling initial examples in Bayesian optimization.
:type str_initial_method_bo: str., optional
:param str_sampling_method_ao: the name of initialization method for
acquisition function optimization.
:type str_sampling_method_ao: str., optional
:param num_samples_ao: the number of samples for acquisition function
optimization. If L-BFGS-B is used as an acquisition function
optimization method, it is employed.
:type num_samples_ao: int., optional
:param str_mlm_method: the name of marginal likelihood maximization
method for Gaussian process regression.
:type str_mlm_method: str., optional
:param seed: None, or random seed.
:type seed: NoneType or int., optional
:returns: tuple of acquired examples, their function values, overall
execution times per iteration, execution time consumed in Gaussian
process regression, and execution time consumed in acquisition
function optimization. Shape: ((`num_init` + `num_iter`, d),
(`num_init` + `num_iter`, 1), (`num_init` + `num_iter`, ), (`num_iter`, ),
(`num_iter`, )), or ((`num_init` + `num_iter`, m, d), (`num_init` + `num_iter`, m, 1),
(`num_init` + `num_iter`, ), (`num_iter`, ), (`num_iter`, )),
where d is a dimensionality of the problem we are solving and m is
a cardinality of sets.
:rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
:raises: AssertionError
"""
assert isinstance(model_bo, bo.BO)
assert callable(fun_target)
assert isinstance(num_init, int)
assert isinstance(num_iter, int)
assert isinstance(str_initial_method_bo, str)
assert isinstance(str_sampling_method_ao, str)
assert isinstance(num_samples_ao, int)
assert isinstance(str_mlm_method, str)
assert isinstance(seed, (int, type(None)))
assert str_initial_method_bo in constants.ALLOWED_INITIALIZING_METHOD_BO
assert str_mlm_method in constants.ALLOWED_MLM_METHOD
model_bo.logger.info('range_X:\n%s',
utils_logger.get_str_array(model_bo.range_X))
model_bo.logger.info('str_cov: %s', model_bo.str_cov)
model_bo.logger.info('str_acq: %s', model_bo.str_acq)
model_bo.logger.info('str_optimizer_method_gp: %s',
model_bo.str_optimizer_method_gp)
model_bo.logger.info('str_optimizer_method_bo: %s',
model_bo.str_optimizer_method_bo)
model_bo.logger.info('str_modelselection_method: %s',
model_bo.str_modelselection_method)
model_bo.logger.info('num_init: %d', num_init)
model_bo.logger.info('num_iter: %d', num_iter)
model_bo.logger.info('str_initial_method_bo: %s', str_initial_method_bo)
model_bo.logger.info('str_sampling_method_ao: %s', str_sampling_method_ao)
model_bo.logger.info('num_samples_ao: %d', num_samples_ao)
model_bo.logger.info('str_mlm_method: %s', str_mlm_method)
model_bo.logger.info('seed: %s', seed)
time_start = time.time()
X_init = model_bo.get_initials(str_initial_method_bo, num_init, seed=seed)
if model_bo.debug:
model_bo.logger.debug('X_init:\n%s',
utils_logger.get_str_array(X_init))
X_final, Y_final, time_all_final, time_surrogate_final, time_acq_final \
= run_single_round_with_initial_inputs(
model_bo, fun_target, X_init, num_iter,
str_sampling_method_ao=str_sampling_method_ao,
num_samples_ao=num_samples_ao,
str_mlm_method=str_mlm_method
)
time_end = time.time()
if model_bo.debug:
model_bo.logger.debug(
'overall time consumed including initializations: %.4f sec.',
time_end - time_start)
return X_final, Y_final, time_all_final, time_surrogate_final, time_acq_final
def run_single_round_with_all_initial_information(
model_bo: bo.BO,
fun_target: constants.TYPING_CALLABLE,
X_train: np.ndarray,
Y_train: np.ndarray,
num_iter: int,
str_sampling_method_ao: str = constants.STR_SAMPLING_METHOD_AO,
num_samples_ao: int = constants.NUM_SAMPLES_AO,
str_mlm_method: str = constants.STR_MLM_METHOD
) -> constants.TYPING_TUPLE_FIVE_ARRAYS:
"""
It optimizes `fun_target` for `num_iter` iterations with given `model_bo`.
It returns the optimization results and execution times.
:param model_bo: Bayesian optimization model.
:type model_bo: bayeso.bo.BO
:param fun_target: a target function.
:type fun_target: callable
:param X_train: initial inputs. Shape: (n, d) or (n, m, d).
:type X_train: numpy.ndarray
:param Y_train: initial outputs. Shape: (n, 1).
:type Y_train: numpy.ndarray
:param num_iter: the number of iterations for Bayesian optimization.
:type num_iter: int.
:param str_sampling_method_ao: the name of initialization method for
acquisition function optimization.
:type str_sampling_method_ao: str., optional
:param num_samples_ao: the number of samples for acquisition function
optimization. If L-BFGS-B is used as an acquisition function
optimization method, it is employed.
:type num_samples_ao: int., optional
:param str_mlm_method: the name of marginal likelihood maximization
method for Gaussian process regression.
:type str_mlm_method: str., optional
:returns: tuple of acquired examples, their function values, overall
execution times per iteration, execution time consumed in Gaussian
process regression, and execution time consumed in acquisition
function optimization. Shape: ((n + `num_iter`, d), (n + `num_iter`, 1),
(`num_iter`, ), (`num_iter`, ), (`num_iter`, )), or ((n + `num_iter`, m, d),
(n + `num_iter`, m, 1), (`num_iter`, ), (`num_iter`, ), (`num_iter`, )).
:rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
:raises: AssertionError
"""
assert isinstance(model_bo, bo.BO)
assert callable(fun_target)
assert isinstance(X_train, np.ndarray)
assert isinstance(Y_train, np.ndarray)
assert isinstance(num_iter, int)
assert isinstance(str_sampling_method_ao, str)
assert isinstance(num_samples_ao, int)
assert isinstance(str_mlm_method, str)
assert len(X_train.shape) == 2
assert len(Y_train.shape) == 2
assert X_train.shape[0] == Y_train.shape[0]
assert Y_train.shape[1] == 1
assert str_mlm_method in constants.ALLOWED_MLM_METHOD
time_start = time.time()
X_final = X_train
Y_final = Y_train
time_all_final = []
time_surrogate_final = []
time_acq_final = []
for ind_iter in range(0, num_iter):
model_bo.logger.info('Iteration %d', ind_iter + 1)
time_iter_start = time.time()
next_point, dict_info = model_bo.optimize(
X_final,
Y_final,
str_sampling_method=str_sampling_method_ao,
num_samples=num_samples_ao,
str_mlm_method=str_mlm_method)
next_points = dict_info['next_points']
acquisitions = dict_info['acquisitions']
time_surrogate = dict_info['time_surrogate']
time_acq = dict_info['time_acq']
if model_bo.debug:
model_bo.logger.debug('next_point: %s',
utils_logger.get_str_array(next_point))
if np.where(np.linalg.norm(next_point - X_final, axis=1)\
< constants.TOLERANCE_DUPLICATED_ACQ)[0].shape[0] > 0: # pragma: no cover
next_point = utils_bo.get_next_best_acquisition(
next_points, acquisitions, X_final)
if model_bo.debug:
model_bo.logger.debug(
'next_point is repeated, so next best is selected.\
next_point: %s',
utils_logger.get_str_array(next_point))
X_final = np.vstack((X_final, next_point))
time_to_evaluate_start = time.time()
Y_final = np.vstack((Y_final, fun_target(next_point)))
time_to_evaluate_end = time.time()
if model_bo.debug:
model_bo.logger.debug(
'time consumed to evaluate: %.4f sec.',
time_to_evaluate_end - time_to_evaluate_start)
time_iter_end = time.time()
time_all_final.append(time_iter_end - time_iter_start)
time_surrogate_final.append(time_surrogate)
time_acq_final.append(time_acq)
time_end = time.time()
if model_bo.debug:
model_bo.logger.debug(
'overall time consumed in single BO round: %.4f sec.',
time_end - time_start)
time_all_final = np.array(time_all_final)
time_surrogate_final = np.array(time_surrogate_final)
time_acq_final = np.array(time_acq_final)
return X_final, Y_final, time_all_final, time_surrogate_final, time_acq_final
def optimize_many_(model_bo,
fun_target,
X_train,
Y_train,
int_iter,
str_initial_method_ao=constants.STR_AO_INITIALIZATION,
int_samples_ao=constants.NUM_ACQ_SAMPLES,
str_mlm_method=constants.STR_MLM_METHOD):
"""
It optimizes `fun_target` for `int_iter` iterations with given `model_bo`.
It returns the optimization results and execution times.
:param model_bo: Bayesian optimization model.
:type model_bo: bayeso.bo.BO
:param fun_target: a target function.
:type fun_target: function
:param X_train: initial inputs. Shape: (n, d) or (n, m, d).
:type X_train: numpy.ndarray
:param Y_train: initial outputs. Shape: (n, 1).
:type Y_train: numpy.ndarray
:param int_iter: the number of iterations for Bayesian optimization.
:type int_iter: int.
:param str_initial_method_ao: the name of initialization method for acquisition function optimization.
:type str_initial_method_ao: str., optional
:param int_samples_ao: the number of samples for acquisition function optimization. If L-BFGS-B is used as an acquisition function optimization method, it is employed.
:type int_samples_ao: int., optional
:param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression.
:type str_mlm_method: str., optional
:returns: tuple of acquired examples, their function values, overall execution times per iteration, execution time consumed in Gaussian process regression, and execution time consumed in acquisition function optimization. Shape: ((n + `int_iter`, d), (n + `int_iter`, 1), (`int_iter`, ), (`int_iter`, ), (`int_iter`, )), or ((n + `int_iter`, m, d), (n + `int_iter`, m, 1), (`int_iter`, ), (`int_iter`, ), (`int_iter`, )).
:rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
:raises: AssertionError
"""
assert isinstance(model_bo, bo.BO)
assert callable(fun_target)
assert isinstance(X_train, np.ndarray)
assert isinstance(Y_train, np.ndarray)
assert isinstance(int_iter, int)
assert isinstance(str_initial_method_ao, str)
assert isinstance(int_samples_ao, int)
assert isinstance(str_mlm_method, str)
assert len(X_train.shape) == 2
assert len(Y_train.shape) == 2
assert X_train.shape[0] == Y_train.shape[0]
assert Y_train.shape[1] == 1
assert str_mlm_method in constants.ALLOWED_MLM_METHOD
time_start = time.time()
X_final = X_train
Y_final = Y_train
time_all_final = []
time_gp_final = []
time_acq_final = []
for ind_iter in range(0, int_iter):
logger.info('Iteration {}'.format(ind_iter + 1))
time_iter_start = time.time()
next_point, dict_info = model_bo.optimize(
X_final,
Y_final,
str_initial_method_ao=str_initial_method_ao,
int_samples=int_samples_ao,
str_mlm_method=str_mlm_method)
next_points = dict_info['next_points']
acquisitions = dict_info['acquisitions']
time_gp = dict_info['time_gp']
time_acq = dict_info['time_acq']
if model_bo.debug:
logger.debug('next_point: {}'.format(
utils_logger.get_str_array(next_point)))
# TODO: check this code, which uses norm.
# if np.where(np.sum(next_point == X_final, axis=1) == X_final.shape[1])[0].shape[0] > 0:
if np.where(np.linalg.norm(next_point - X_final, axis=1) < 1e-3
)[0].shape[0] > 0: # pragma: no cover
next_point = get_next_best_acquisition(next_points, acquisitions,
X_final)
if model_bo.debug:
logger.debug(
'next_point is repeated, so next best is selected. next_point: {}'
.format(utils_logger.get_str_array(next_point)))
X_final = np.vstack((X_final, next_point))
time_to_evaluate_start = time.time()
Y_final = np.vstack((Y_final, fun_target(next_point)))
time_to_evaluate_end = time.time()
if model_bo.debug:
logger.debug('time consumed to evaluate: {:.4f} sec.'.format(
time_to_evaluate_end - time_to_evaluate_start))
time_iter_end = time.time()
time_all_final.append(time_iter_end - time_iter_start)
time_gp_final.append(time_gp)
time_acq_final.append(time_acq)
time_end = time.time()
if model_bo.debug:
logger.debug(
'overall time consumed in single BO round: {:.4f} sec.'.format(
time_end - time_start))
time_all_final = np.array(time_all_final)
time_gp_final = np.array(time_gp_final)
time_acq_final = np.array(time_acq_final)
return X_final, Y_final, time_all_final, time_gp_final, time_acq_final
def optimize_many_with_random_init(
model_bo,
fun_target,
int_init,
int_iter,
str_initial_method_bo=constants.STR_BO_INITIALIZATION,
str_initial_method_ao=constants.STR_AO_INITIALIZATION,
int_samples_ao=constants.NUM_ACQ_SAMPLES,
str_mlm_method=constants.STR_MLM_METHOD,
int_seed=None):
"""
It optimizes `fun_target` for `int_iter` iterations with given `model_bo` and `int_init` initial examples.
Initial examples are sampled by `get_initial` method in `model_bo`.
It returns the optimization results and execution times.
:param model_bo: Bayesian optimization model.
:type model_bo: bayeso.bo.BO
:param fun_target: a target function.
:type fun_target: function
:param int_init: the number of initial examples for Bayesian optimization.
:type int_init: int.
:param int_iter: the number of iterations for Bayesian optimization.
:type int_iter: int.
:param str_initial_method_bo: the name of initialization method for sampling initial examples in Bayesian optimization.
:type str_initial_method_bo: str., optional
:param str_initial_method_ao: the name of initialization method for acquisition function optimization.
:type str_initial_method_ao: str., optional
:param int_samples_ao: the number of samples for acquisition function optimization. If L-BFGS-B is used as an acquisition function optimization method, it is employed.
:type int_samples_ao: int., optional
:param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression.
:type str_mlm_method: str., optional
:param int_seed: None, or random seed.
:type int_seed: NoneType or int., optional
:returns: tuple of acquired examples, their function values, overall execution times per iteration, execution time consumed in Gaussian process regression, and execution time consumed in acquisition function optimization. Shape: ((`int_init` + `int_iter`, d), (`int_init` + `int_iter`, 1), (`int_init` + `int_iter`, ), (`int_iter`, ), (`int_iter`, )), or ((`int_init` + `int_iter`, m, d), (`int_init` + `int_iter`, m, 1), (`int_init` + `int_iter`, ), (`int_iter`, ), (`int_iter`, )), where d is a dimensionality of the problem we are solving and m is a cardinality of sets.
:rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray)
:raises: AssertionError
"""
assert isinstance(model_bo, bo.BO)
assert callable(fun_target)
assert isinstance(int_init, int)
assert isinstance(int_iter, int)
assert isinstance(str_initial_method_bo, str)
assert isinstance(str_initial_method_ao, str)
assert isinstance(int_samples_ao, int)
assert isinstance(str_mlm_method, str)
assert isinstance(int_seed, int) or int_seed is None
assert str_initial_method_bo in constants.ALLOWED_INITIALIZATIONS_BO
assert str_mlm_method in constants.ALLOWED_MLM_METHOD
logger.info('arr_range:\n{}'.format(
utils_logger.get_str_array(model_bo.arr_range)))
logger.info('str_cov: {}'.format(model_bo.str_cov))
logger.info('str_acq: {}'.format(model_bo.str_acq))
logger.info('str_optimizer_method_gp: {}'.format(
model_bo.str_optimizer_method_gp))
logger.info('str_optimizer_method_bo: {}'.format(
model_bo.str_optimizer_method_bo))
logger.info('str_modelselection_method: {}'.format(
model_bo.str_modelselection_method))
logger.info('int_init: {}'.format(int_init))
logger.info('int_iter: {}'.format(int_iter))
logger.info('str_initial_method_bo: {}'.format(str_initial_method_bo))
logger.info('str_initial_method_ao: {}'.format(str_initial_method_ao))
logger.info('int_samples_ao: {}'.format(int_samples_ao))
logger.info('str_mlm_method: {}'.format(str_mlm_method))
logger.info('int_seed: {}'.format(int_seed))
time_start = time.time()
X_init = model_bo.get_initial(str_initial_method_bo,
fun_objective=fun_target,
int_samples=int_init,
int_seed=int_seed)
if model_bo.debug:
logger.debug('X_init:\n{}'.format(utils_logger.get_str_array(X_init)))
X_final, Y_final, time_all_final, time_gp_final, time_acq_final = optimize_many(
model_bo,
fun_target,
X_init,
int_iter,
str_initial_method_ao=str_initial_method_ao,
int_samples_ao=int_samples_ao,
str_mlm_method=str_mlm_method)
time_end = time.time()
if model_bo.debug:
logger.debug(
'overall time consumed including initializations: {:.4f} sec.'.
format(time_end - time_start))
return X_final, Y_final, time_all_final, time_gp_final, time_acq_final