def test_get_str_array(): with pytest.raises(AssertionError) as error: utils_logger.get_str_array(123) with pytest.raises(AssertionError) as error: utils_logger.get_str_array(12.3) str_ = utils_logger.get_str_array(np.array([1, 2, 3])) print(str_) assert str_ == '[1, 2, 3]' str_ = utils_logger.get_str_array(np.array([1.1, 2.5, 3.0])) print(str_) assert str_ == '[1.100, 2.500, 3.000]' str_ = utils_logger.get_str_array(np.array([[1, 2, 3], [2, 2, 2]])) print(str_) assert str_ == '[[1, 2, 3],\n[2, 2, 2]]' str_ = utils_logger.get_str_array( np.array([[1.1, 2.2, 3.33], [2.2, 2.4, 2.9]])) print(str_) assert str_ == '[[1.100, 2.200, 3.330],\n[2.200, 2.400, 2.900]]' str_ = utils_logger.get_str_array( np.array([[[1, 2, 3], [2, 2, 2]], [[1, 2, 3], [2, 2, 2]]])) print(str_) assert str_ == '[[[1, 2, 3],\n[2, 2, 2]],\n[[1, 2, 3],\n[2, 2, 2]]]' str_ = utils_logger.get_str_array( np.array([[[1.1, 2.2, 3.33], [2.2, 2.4, 2.9]], [[1.1, 2.2, 3.33], [2.2, 2.4, 2.9]]])) print(str_) assert str_ == '[[[1.100, 2.200, 3.330],\n[2.200, 2.400, 2.900]],\n[[1.100, 2.200, 3.330],\n[2.200, 2.400, 2.900]]]'
def get_samples( self, str_sampling_method: str, fun_objective: constants.TYPING_UNION_CALLABLE_NONE = None, num_samples: int = constants.NUM_SAMPLES_AO, seed: constants.TYPING_UNION_INT_NONE = None, ) -> np.ndarray: """ It returns `num_samples` examples, sampled by a sampling method `str_sampling_method`. :param str_sampling_method: the name of sampling method. :type str_sampling_method: str. :param fun_objective: None, or objective function. :type fun_objective: NoneType or callable, optional :param num_samples: the number of samples. :type num_samples: int., optional :param seed: None, or random seed. :type seed: NoneType or int., optional :returns: sampled examples. Shape: (`num_samples`, d). :rtype: numpy.ndarray :raises: AssertionError """ assert isinstance(str_sampling_method, str) assert callable(fun_objective) or fun_objective is None assert isinstance(num_samples, int) assert isinstance(seed, (int, constants.TYPE_NONE)) assert str_sampling_method in constants.ALLOWED_SAMPLING_METHOD if str_sampling_method == 'grid': assert fun_objective is not None if self.debug: self.logger.debug( 'For this option, num_samples is used as num_grids.') samples = self._get_samples_grid(num_grids=num_samples) samples = utils_bo.get_best_acquisition_by_evaluation( samples, fun_objective) elif str_sampling_method == 'uniform': samples = self._get_samples_uniform(num_samples, seed=seed) elif str_sampling_method == 'gaussian': samples = self._get_samples_gaussian(num_samples, seed=seed) elif str_sampling_method == 'sobol': samples = self._get_samples_sobol(num_samples, seed=seed) elif str_sampling_method == 'halton': samples = self._get_samples_halton(num_samples, seed=seed) else: raise NotImplementedError( 'get_samples: allowed str_sampling_method,\ but it is not implemented.') if self.debug: self.logger.debug('samples:\n%s', utils_logger.get_str_array(samples)) return samples
def get_initial( self, str_initial_method, fun_objective=None, int_samples=constants.NUM_ACQ_SAMPLES, int_seed=None, ): """ It returns a single example or `int_samples` examples, sampled by a certian method `str_initial_method`. :param str_initial_method: the name of sampling method. :type str_initial_method: str. :param fun_objective: None, or objective function. :type fun_objective: NoneType or function, optional :param int_samples: the number of samples. :type int_samples: int., optional :param int_seed: None, or random seed. :type int_seed: NoneType or int., optional :returns: sampled examples. Shape: (1, d) or (`int_samples`, d). :rtype: numpy.ndarray :raises: AssertionError """ assert isinstance(str_initial_method, str) assert callable(fun_objective) or fun_objective is None assert isinstance(int_samples, int) assert isinstance(int_seed, int) or int_seed is None if str_initial_method == 'grid': assert fun_objective is not None if self.debug: logger.debug('int_samples is ignored, because grid is chosen.') arr_initials = self._get_initial_grid() arr_initials = get_best_acquisition(arr_initials, fun_objective) elif str_initial_method == 'uniform': arr_initials = self._get_initial_uniform(int_samples, int_seed=int_seed) elif str_initial_method == 'sobol': arr_initials = self._get_initial_sobol(int_samples, int_seed=int_seed) elif str_initial_method == 'latin': raise NotImplementedError('get_initial: latin') else: raise NotImplementedError( 'get_initial: allowed str_initial_method, but it is not implemented.' ) if self.debug: logger.debug('arr_initials:\n{}'.format( utils_logger.get_str_array(arr_initials))) return arr_initials
def optimize(self, num_init: int, seed: constants.TYPING_UNION_INT_NONE=None, ) -> constants.TYPING_TUPLE_FIVE_ARRAYS: """ It returns the optimization results and times consumed, given the number of initial samples `num_init` and a random seed `seed`. :param num_init: the number of initial samples. :type num_init: int. :param seed: None, or a random seed. :type seed: NoneType or int., optional :returns: a tuple of acquired samples, their function values, overall times consumed per iteration, time consumed in modeling Gaussian process regression, and time consumed in acquisition function optimization. Shape: ((`num_init` + `num_iter`, d), (`num_init` + `num_iter`, 1), (`num_init` + `num_iter`, ), (`num_iter`, ), (`num_iter`, )), or ((`num_init` + `num_iter`, m, d), (`num_init` + `num_iter`, m, 1), (`num_init` + `num_iter`, ), (`num_iter`, ), (`num_iter`, )), where d is a dimensionality of the problem we are solving and m is a cardinality of sets. :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) :raises: AssertionError """ assert isinstance(num_init, int) assert isinstance(seed, (int, type(None))) assert num_init > 0 self.print_info(num_init, seed) time_start = time.time() X_init = self.model_bo.get_initials(self.str_initial_method_bo, num_init, seed=seed) if self.debug: self.model_bo.logger.debug('X_init:\n%s', utils_logger.get_str_array(X_init)) X, Y, time_all, time_surrogate, time_acq = self.optimize_with_initial_inputs(X_init) time_end = time.time() if self.debug: self.model_bo.logger.debug('overall time consumed including initializations: %.4f sec.', time_end - time_start) return X, Y, time_all, time_surrogate, time_acq
def print_info(self, num_init, seed): """ It returns the optimization results and times consumed, given inital inputs `X`. :param num_init: the number of initial points. :type num_init: int. :param seed: a random seed. :type seed: int. :returns: None :rtype: NoneType """ self.model_bo.logger.info('====================') self.model_bo.logger.info('range_X:\n%s', utils_logger.get_str_array(self.range_X)) self.model_bo.logger.info('num_init: %d', num_init) self.model_bo.logger.info('num_iter: %d', self.num_iter) self.model_bo.logger.info('str_surrogate: %s', self.str_surrogate) if self.str_surrogate in constants.ALLOWED_SURROGATE: self.model_bo.logger.info('str_cov: %s', self.str_cov) self.model_bo.logger.info('str_acq: %s', self.str_acq) self.model_bo.logger.info('normalize_Y: %s', self.normalize_Y) if self.str_surrogate in constants.ALLOWED_SURROGATE: self.model_bo.logger.info('use_ard: %s', self.use_ard) self.model_bo.logger.info('str_initial_method_bo: %s', self.str_initial_method_bo) self.model_bo.logger.info('str_sampling_method_ao: %s', self.str_sampling_method_ao) if self.str_surrogate in ['gp']: self.model_bo.logger.info('str_optimizer_method_gp: %s', self.str_optimizer_method_gp) if self.str_surrogate in ['tp']: self.model_bo.logger.info('str_optimizer_method_tp: %s', self.str_optimizer_method_tp) self.model_bo.logger.info('str_optimizer_method_bo: %s', self.str_optimizer_method_bo) if self.str_surrogate in ['gp']: self.model_bo.logger.info('str_mlm_method: %s', self.str_mlm_method) self.model_bo.logger.info('str_modelselection_method: %s', self.str_modelselection_method) self.model_bo.logger.info('num_samples_ao: %d', self.num_samples_ao) self.model_bo.logger.info('seed: %s', seed) self.model_bo.logger.info('debug: %s', self.debug) self.model_bo.logger.info('====================')
def _get_next_best_sample(self, next_sample: np.ndarray, X: np.ndarray, next_samples: np.ndarray, acq_vals: np.ndarray, ) -> np.ndarray: """ It returns the next best sample in terms of acquisition function values. :param next_sample: the next sample acquired. :type next_sample: np.ndarray :param X: the samples evaluated so far. :type X: np.ndarray :param next_samples: the candidates of the next sample. :type next_samples: np.ndarray :param acq_vals: the values of acquisition function over `next_samples`. :type acq_vals: np.ndarray :returns: the next best sample. Shape: (d, ). :rtype: numpy.ndarray :raises: AssertionError """ assert isinstance(next_sample, np.ndarray) assert isinstance(X, np.ndarray) assert isinstance(next_samples, np.ndarray) assert isinstance(acq_vals, np.ndarray) if np.where(np.linalg.norm(next_sample - X, axis=1)\ < constants.TOLERANCE_DUPLICATED_ACQ)[0].shape[0] > 0: # pragma: no cover next_sample = utils_bo.get_next_best_acquisition( next_samples, acq_vals, X) if self.debug: self.model_bo.logger.debug('next_sample is repeated, so next best is selected.\ next_sample: %s', utils_logger.get_str_array(next_sample)) return next_sample
def _optimize(self, fun_negative_acquisition: constants.TYPING_CALLABLE, str_sampling_method: str, num_samples: int) -> constants.TYPING_TUPLE_TWO_ARRAYS: """ It optimizes `fun_negative_function` with `self.str_optimizer_method_bo`. `num_samples` examples are determined by `str_sampling_method`, to start acquisition function optimization. :param fun_objective: negative acquisition function. :type fun_objective: callable :param str_sampling_method: the name of sampling method. :type str_sampling_method: str. :param num_samples: the number of samples. :type num_samples: int. :returns: tuple of next point to evaluate and all candidates determined by acquisition function optimization. Shape: ((d, ), (`num_samples`, d)). :rtype: (numpy.ndarray, numpy.ndarray) """ list_next_point = [] if self.str_optimizer_method_bo == 'L-BFGS-B': list_bounds = self._get_bounds() initials = self.get_samples(str_sampling_method, fun_objective=fun_negative_acquisition, num_samples=num_samples) for arr_initial in initials: next_point = minimize(fun_negative_acquisition, x0=arr_initial, bounds=list_bounds, method=self.str_optimizer_method_bo, options={'disp': False}) next_point_x = next_point.x list_next_point.append(next_point_x) if self.debug: self.logger.debug('acquired sample: %s', utils_logger.get_str_array(next_point_x)) elif self.str_optimizer_method_bo == 'DIRECT': # pragma: no cover self.logger.debug('num_samples is ignored.') list_bounds = self._get_bounds() next_point = directminimize( fun_negative_acquisition, bounds=list_bounds, maxf=88888, ) next_point_x = next_point.x list_next_point.append(next_point_x) elif self.str_optimizer_method_bo == 'CMA-ES': self.logger.debug('num_samples is ignored.') list_bounds = self._get_bounds() list_bounds = np.array(list_bounds) def fun_wrapper(f): def g(bx): return f(bx)[0] return g initials = self.get_samples(str_sampling_method, fun_objective=fun_negative_acquisition, num_samples=1) cur_sigma0 = np.mean(list_bounds[:, 1] - list_bounds[:, 0]) / 4.0 next_point_x = cma.fmin(fun_wrapper(fun_negative_acquisition), initials[0], cur_sigma0, options={ 'bounds': [list_bounds[:, 0], list_bounds[:, 1]], 'verbose': -1, 'maxfevals': 1e5 })[0] list_next_point.append(next_point_x) next_points = np.array(list_next_point) next_point = utils_bo.get_best_acquisition_by_evaluation( next_points, fun_negative_acquisition)[0] return next_point, next_points
def _optimize(self, fun_negative_acquisition, str_initial_method, int_samples): """ It optimizes `fun_negative_function` with `self.str_optimizer_method_bo`. `int_samples` examples are determined by `str_initial_method`, to start acquisition function optimization. :param fun_objective: negative acquisition function. :type fun_objective: function :param str_initial_method: the name of sampling method. :type str_initial_method: str. :param int_samples: the number of samples. :type int_samples: int. :returns: tuple of next point to evaluate and all candidates determined by acquisition function optimization. Shape: ((d, ), (`int_samples`, d)). :rtype: (numpy.ndarray, numpy.ndarray) """ list_next_point = [] if self.str_optimizer_method_bo == 'L-BFGS-B': list_bounds = self._get_bounds() arr_initials = self.get_initial( str_initial_method, fun_objective=fun_negative_acquisition, int_samples=int_samples) for arr_initial in arr_initials: next_point = minimize(fun_negative_acquisition, x0=arr_initial, bounds=list_bounds, method=self.str_optimizer_method_bo, options={'disp': False}) next_point_x = next_point.x list_next_point.append(next_point_x) if self.debug: logger.debug('acquired sample: {}'.format( utils_logger.get_str_array(next_point_x))) elif self.str_optimizer_method_bo == 'DIRECT': # pragma: no cover list_bounds = self._get_bounds() next_point = directminimize( fun_negative_acquisition, bounds=list_bounds, maxf=88888, ) next_point_x = next_point.x list_next_point.append(next_point_x) elif self.str_optimizer_method_bo == 'CMA-ES': list_bounds = self._get_bounds() list_bounds = np.array(list_bounds) def fun_wrapper(f): def g(bx): return f(bx)[0] return g arr_initials = self.get_initial( str_initial_method, fun_objective=fun_negative_acquisition, int_samples=1) cur_sigma0 = np.mean(list_bounds[:, 1] - list_bounds[:, 0]) / 4.0 next_point_x = cma.fmin(fun_wrapper(fun_negative_acquisition), arr_initials[0], cur_sigma0, options={ 'bounds': [list_bounds[:, 0], list_bounds[:, 1]], 'verbose': -1, 'maxfevals': 1e5 })[0] list_next_point.append(next_point_x) next_points = np.array(list_next_point) next_point = get_best_acquisition(next_points, fun_negative_acquisition) return next_point.flatten(), next_points
def optimize_with_all_initial_information(self, X: np.ndarray, Y: np.ndarray, ) -> constants.TYPING_TUPLE_FIVE_ARRAYS: """ It returns the optimization results and times consumed, given inital inputs `X` and their corresponding outputs `Y`. :param X: initial inputs. Shape: (n, d) or (n, m, d). :type X: numpy.ndarray :param Y: initial outputs. Shape: (n, 1). :type Y: numpy.ndarray :returns: a tuple of acquired samples, their function values, overall times consumed per iteration, time consumed in modeling Gaussian process regression, and time consumed in acquisition function optimization. Shape: ((n + `num_iter`, d), (n + `num_iter`, 1), (`num_iter`, ), (`num_iter`, ), (`num_iter`, )), or ((n + `num_iter`, m, d), (n + `num_iter`, m, 1), (`num_iter`, ), (`num_iter`, ), (`num_iter`, )). :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) :raises: AssertionError """ assert isinstance(X, np.ndarray) assert isinstance(Y, np.ndarray) assert len(X.shape) == 2 assert len(Y.shape) == 2 assert X.shape[0] == Y.shape[0] assert Y.shape[1] == 1 time_start = time.time() X_ = X Y_ = Y time_all_ = [] time_surrogate_ = [] time_acq_ = [] pbar = tqdm(range(0, self.num_iter)) for ind_iter in pbar: self.model_bo.logger.info('Iteration %d', ind_iter + 1) time_iter_start = time.time() next_sample, dict_info = self.optimize_single_iteration(X_, Y_) next_samples = dict_info['next_points'] acq_vals = dict_info['acquisitions'] time_surrogate = dict_info['time_surrogate'] time_acq = dict_info['time_acq'] if self.debug: self.model_bo.logger.debug('next_sample: %s', utils_logger.get_str_array(next_sample)) next_sample = self._get_next_best_sample(next_sample, X_, next_samples, acq_vals) X_ = np.vstack((X_, next_sample)) time_to_evaluate_start = time.time() Y_ = np.vstack((Y_, self.fun_target(next_sample))) time_to_evaluate_end = time.time() if self.debug: self.model_bo.logger.debug('time consumed to evaluate: %.4f sec.', time_to_evaluate_end - time_to_evaluate_start) time_iter_end = time.time() time_all_.append(time_iter_end - time_iter_start) time_surrogate_.append(time_surrogate) time_acq_.append(time_acq) time_end = time.time() if self.debug: self.model_bo.logger.debug('overall time consumed in single BO round: %.4f sec.', time_end - time_start) time_all_ = np.array(time_all_) time_surrogate_ = np.array(time_surrogate_) time_acq_ = np.array(time_acq_) return X_, Y_, time_all_, time_surrogate_, time_acq_
def run_single_round( model_bo: bo.BO, fun_target: constants.TYPING_CALLABLE, num_init: int, num_iter: int, str_initial_method_bo: str = constants.STR_INITIALIZING_METHOD_BO, str_sampling_method_ao: str = constants.STR_SAMPLING_METHOD_AO, num_samples_ao: int = constants.NUM_SAMPLES_AO, str_mlm_method: str = constants.STR_MLM_METHOD, seed: constants.TYPING_UNION_INT_NONE = None ) -> constants.TYPING_TUPLE_FIVE_ARRAYS: """ It optimizes `fun_target` for `num_iter` iterations with given `model_bo` and `num_init` initial examples. Initial examples are sampled by `get_initials` method in `model_bo`. It returns the optimization results and execution times. :param model_bo: Bayesian optimization model. :type model_bo: bayeso.bo.BO :param fun_target: a target function. :type fun_target: callable :param num_init: the number of initial examples for Bayesian optimization. :type num_init: int. :param num_iter: the number of iterations for Bayesian optimization. :type num_iter: int. :param str_initial_method_bo: the name of initialization method for sampling initial examples in Bayesian optimization. :type str_initial_method_bo: str., optional :param str_sampling_method_ao: the name of initialization method for acquisition function optimization. :type str_sampling_method_ao: str., optional :param num_samples_ao: the number of samples for acquisition function optimization. If L-BFGS-B is used as an acquisition function optimization method, it is employed. :type num_samples_ao: int., optional :param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression. :type str_mlm_method: str., optional :param seed: None, or random seed. :type seed: NoneType or int., optional :returns: tuple of acquired examples, their function values, overall execution times per iteration, execution time consumed in Gaussian process regression, and execution time consumed in acquisition function optimization. Shape: ((`num_init` + `num_iter`, d), (`num_init` + `num_iter`, 1), (`num_init` + `num_iter`, ), (`num_iter`, ), (`num_iter`, )), or ((`num_init` + `num_iter`, m, d), (`num_init` + `num_iter`, m, 1), (`num_init` + `num_iter`, ), (`num_iter`, ), (`num_iter`, )), where d is a dimensionality of the problem we are solving and m is a cardinality of sets. :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) :raises: AssertionError """ assert isinstance(model_bo, bo.BO) assert callable(fun_target) assert isinstance(num_init, int) assert isinstance(num_iter, int) assert isinstance(str_initial_method_bo, str) assert isinstance(str_sampling_method_ao, str) assert isinstance(num_samples_ao, int) assert isinstance(str_mlm_method, str) assert isinstance(seed, (int, type(None))) assert str_initial_method_bo in constants.ALLOWED_INITIALIZING_METHOD_BO assert str_mlm_method in constants.ALLOWED_MLM_METHOD model_bo.logger.info('range_X:\n%s', utils_logger.get_str_array(model_bo.range_X)) model_bo.logger.info('str_cov: %s', model_bo.str_cov) model_bo.logger.info('str_acq: %s', model_bo.str_acq) model_bo.logger.info('str_optimizer_method_gp: %s', model_bo.str_optimizer_method_gp) model_bo.logger.info('str_optimizer_method_bo: %s', model_bo.str_optimizer_method_bo) model_bo.logger.info('str_modelselection_method: %s', model_bo.str_modelselection_method) model_bo.logger.info('num_init: %d', num_init) model_bo.logger.info('num_iter: %d', num_iter) model_bo.logger.info('str_initial_method_bo: %s', str_initial_method_bo) model_bo.logger.info('str_sampling_method_ao: %s', str_sampling_method_ao) model_bo.logger.info('num_samples_ao: %d', num_samples_ao) model_bo.logger.info('str_mlm_method: %s', str_mlm_method) model_bo.logger.info('seed: %s', seed) time_start = time.time() X_init = model_bo.get_initials(str_initial_method_bo, num_init, seed=seed) if model_bo.debug: model_bo.logger.debug('X_init:\n%s', utils_logger.get_str_array(X_init)) X_final, Y_final, time_all_final, time_surrogate_final, time_acq_final \ = run_single_round_with_initial_inputs( model_bo, fun_target, X_init, num_iter, str_sampling_method_ao=str_sampling_method_ao, num_samples_ao=num_samples_ao, str_mlm_method=str_mlm_method ) time_end = time.time() if model_bo.debug: model_bo.logger.debug( 'overall time consumed including initializations: %.4f sec.', time_end - time_start) return X_final, Y_final, time_all_final, time_surrogate_final, time_acq_final
def run_single_round_with_all_initial_information( model_bo: bo.BO, fun_target: constants.TYPING_CALLABLE, X_train: np.ndarray, Y_train: np.ndarray, num_iter: int, str_sampling_method_ao: str = constants.STR_SAMPLING_METHOD_AO, num_samples_ao: int = constants.NUM_SAMPLES_AO, str_mlm_method: str = constants.STR_MLM_METHOD ) -> constants.TYPING_TUPLE_FIVE_ARRAYS: """ It optimizes `fun_target` for `num_iter` iterations with given `model_bo`. It returns the optimization results and execution times. :param model_bo: Bayesian optimization model. :type model_bo: bayeso.bo.BO :param fun_target: a target function. :type fun_target: callable :param X_train: initial inputs. Shape: (n, d) or (n, m, d). :type X_train: numpy.ndarray :param Y_train: initial outputs. Shape: (n, 1). :type Y_train: numpy.ndarray :param num_iter: the number of iterations for Bayesian optimization. :type num_iter: int. :param str_sampling_method_ao: the name of initialization method for acquisition function optimization. :type str_sampling_method_ao: str., optional :param num_samples_ao: the number of samples for acquisition function optimization. If L-BFGS-B is used as an acquisition function optimization method, it is employed. :type num_samples_ao: int., optional :param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression. :type str_mlm_method: str., optional :returns: tuple of acquired examples, their function values, overall execution times per iteration, execution time consumed in Gaussian process regression, and execution time consumed in acquisition function optimization. Shape: ((n + `num_iter`, d), (n + `num_iter`, 1), (`num_iter`, ), (`num_iter`, ), (`num_iter`, )), or ((n + `num_iter`, m, d), (n + `num_iter`, m, 1), (`num_iter`, ), (`num_iter`, ), (`num_iter`, )). :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) :raises: AssertionError """ assert isinstance(model_bo, bo.BO) assert callable(fun_target) assert isinstance(X_train, np.ndarray) assert isinstance(Y_train, np.ndarray) assert isinstance(num_iter, int) assert isinstance(str_sampling_method_ao, str) assert isinstance(num_samples_ao, int) assert isinstance(str_mlm_method, str) assert len(X_train.shape) == 2 assert len(Y_train.shape) == 2 assert X_train.shape[0] == Y_train.shape[0] assert Y_train.shape[1] == 1 assert str_mlm_method in constants.ALLOWED_MLM_METHOD time_start = time.time() X_final = X_train Y_final = Y_train time_all_final = [] time_surrogate_final = [] time_acq_final = [] for ind_iter in range(0, num_iter): model_bo.logger.info('Iteration %d', ind_iter + 1) time_iter_start = time.time() next_point, dict_info = model_bo.optimize( X_final, Y_final, str_sampling_method=str_sampling_method_ao, num_samples=num_samples_ao, str_mlm_method=str_mlm_method) next_points = dict_info['next_points'] acquisitions = dict_info['acquisitions'] time_surrogate = dict_info['time_surrogate'] time_acq = dict_info['time_acq'] if model_bo.debug: model_bo.logger.debug('next_point: %s', utils_logger.get_str_array(next_point)) if np.where(np.linalg.norm(next_point - X_final, axis=1)\ < constants.TOLERANCE_DUPLICATED_ACQ)[0].shape[0] > 0: # pragma: no cover next_point = utils_bo.get_next_best_acquisition( next_points, acquisitions, X_final) if model_bo.debug: model_bo.logger.debug( 'next_point is repeated, so next best is selected.\ next_point: %s', utils_logger.get_str_array(next_point)) X_final = np.vstack((X_final, next_point)) time_to_evaluate_start = time.time() Y_final = np.vstack((Y_final, fun_target(next_point))) time_to_evaluate_end = time.time() if model_bo.debug: model_bo.logger.debug( 'time consumed to evaluate: %.4f sec.', time_to_evaluate_end - time_to_evaluate_start) time_iter_end = time.time() time_all_final.append(time_iter_end - time_iter_start) time_surrogate_final.append(time_surrogate) time_acq_final.append(time_acq) time_end = time.time() if model_bo.debug: model_bo.logger.debug( 'overall time consumed in single BO round: %.4f sec.', time_end - time_start) time_all_final = np.array(time_all_final) time_surrogate_final = np.array(time_surrogate_final) time_acq_final = np.array(time_acq_final) return X_final, Y_final, time_all_final, time_surrogate_final, time_acq_final
def optimize_many_(model_bo, fun_target, X_train, Y_train, int_iter, str_initial_method_ao=constants.STR_AO_INITIALIZATION, int_samples_ao=constants.NUM_ACQ_SAMPLES, str_mlm_method=constants.STR_MLM_METHOD): """ It optimizes `fun_target` for `int_iter` iterations with given `model_bo`. It returns the optimization results and execution times. :param model_bo: Bayesian optimization model. :type model_bo: bayeso.bo.BO :param fun_target: a target function. :type fun_target: function :param X_train: initial inputs. Shape: (n, d) or (n, m, d). :type X_train: numpy.ndarray :param Y_train: initial outputs. Shape: (n, 1). :type Y_train: numpy.ndarray :param int_iter: the number of iterations for Bayesian optimization. :type int_iter: int. :param str_initial_method_ao: the name of initialization method for acquisition function optimization. :type str_initial_method_ao: str., optional :param int_samples_ao: the number of samples for acquisition function optimization. If L-BFGS-B is used as an acquisition function optimization method, it is employed. :type int_samples_ao: int., optional :param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression. :type str_mlm_method: str., optional :returns: tuple of acquired examples, their function values, overall execution times per iteration, execution time consumed in Gaussian process regression, and execution time consumed in acquisition function optimization. Shape: ((n + `int_iter`, d), (n + `int_iter`, 1), (`int_iter`, ), (`int_iter`, ), (`int_iter`, )), or ((n + `int_iter`, m, d), (n + `int_iter`, m, 1), (`int_iter`, ), (`int_iter`, ), (`int_iter`, )). :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) :raises: AssertionError """ assert isinstance(model_bo, bo.BO) assert callable(fun_target) assert isinstance(X_train, np.ndarray) assert isinstance(Y_train, np.ndarray) assert isinstance(int_iter, int) assert isinstance(str_initial_method_ao, str) assert isinstance(int_samples_ao, int) assert isinstance(str_mlm_method, str) assert len(X_train.shape) == 2 assert len(Y_train.shape) == 2 assert X_train.shape[0] == Y_train.shape[0] assert Y_train.shape[1] == 1 assert str_mlm_method in constants.ALLOWED_MLM_METHOD time_start = time.time() X_final = X_train Y_final = Y_train time_all_final = [] time_gp_final = [] time_acq_final = [] for ind_iter in range(0, int_iter): logger.info('Iteration {}'.format(ind_iter + 1)) time_iter_start = time.time() next_point, dict_info = model_bo.optimize( X_final, Y_final, str_initial_method_ao=str_initial_method_ao, int_samples=int_samples_ao, str_mlm_method=str_mlm_method) next_points = dict_info['next_points'] acquisitions = dict_info['acquisitions'] time_gp = dict_info['time_gp'] time_acq = dict_info['time_acq'] if model_bo.debug: logger.debug('next_point: {}'.format( utils_logger.get_str_array(next_point))) # TODO: check this code, which uses norm. # if np.where(np.sum(next_point == X_final, axis=1) == X_final.shape[1])[0].shape[0] > 0: if np.where(np.linalg.norm(next_point - X_final, axis=1) < 1e-3 )[0].shape[0] > 0: # pragma: no cover next_point = get_next_best_acquisition(next_points, acquisitions, X_final) if model_bo.debug: logger.debug( 'next_point is repeated, so next best is selected. next_point: {}' .format(utils_logger.get_str_array(next_point))) X_final = np.vstack((X_final, next_point)) time_to_evaluate_start = time.time() Y_final = np.vstack((Y_final, fun_target(next_point))) time_to_evaluate_end = time.time() if model_bo.debug: logger.debug('time consumed to evaluate: {:.4f} sec.'.format( time_to_evaluate_end - time_to_evaluate_start)) time_iter_end = time.time() time_all_final.append(time_iter_end - time_iter_start) time_gp_final.append(time_gp) time_acq_final.append(time_acq) time_end = time.time() if model_bo.debug: logger.debug( 'overall time consumed in single BO round: {:.4f} sec.'.format( time_end - time_start)) time_all_final = np.array(time_all_final) time_gp_final = np.array(time_gp_final) time_acq_final = np.array(time_acq_final) return X_final, Y_final, time_all_final, time_gp_final, time_acq_final
def optimize_many_with_random_init( model_bo, fun_target, int_init, int_iter, str_initial_method_bo=constants.STR_BO_INITIALIZATION, str_initial_method_ao=constants.STR_AO_INITIALIZATION, int_samples_ao=constants.NUM_ACQ_SAMPLES, str_mlm_method=constants.STR_MLM_METHOD, int_seed=None): """ It optimizes `fun_target` for `int_iter` iterations with given `model_bo` and `int_init` initial examples. Initial examples are sampled by `get_initial` method in `model_bo`. It returns the optimization results and execution times. :param model_bo: Bayesian optimization model. :type model_bo: bayeso.bo.BO :param fun_target: a target function. :type fun_target: function :param int_init: the number of initial examples for Bayesian optimization. :type int_init: int. :param int_iter: the number of iterations for Bayesian optimization. :type int_iter: int. :param str_initial_method_bo: the name of initialization method for sampling initial examples in Bayesian optimization. :type str_initial_method_bo: str., optional :param str_initial_method_ao: the name of initialization method for acquisition function optimization. :type str_initial_method_ao: str., optional :param int_samples_ao: the number of samples for acquisition function optimization. If L-BFGS-B is used as an acquisition function optimization method, it is employed. :type int_samples_ao: int., optional :param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression. :type str_mlm_method: str., optional :param int_seed: None, or random seed. :type int_seed: NoneType or int., optional :returns: tuple of acquired examples, their function values, overall execution times per iteration, execution time consumed in Gaussian process regression, and execution time consumed in acquisition function optimization. Shape: ((`int_init` + `int_iter`, d), (`int_init` + `int_iter`, 1), (`int_init` + `int_iter`, ), (`int_iter`, ), (`int_iter`, )), or ((`int_init` + `int_iter`, m, d), (`int_init` + `int_iter`, m, 1), (`int_init` + `int_iter`, ), (`int_iter`, ), (`int_iter`, )), where d is a dimensionality of the problem we are solving and m is a cardinality of sets. :rtype: (numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray, numpy.ndarray) :raises: AssertionError """ assert isinstance(model_bo, bo.BO) assert callable(fun_target) assert isinstance(int_init, int) assert isinstance(int_iter, int) assert isinstance(str_initial_method_bo, str) assert isinstance(str_initial_method_ao, str) assert isinstance(int_samples_ao, int) assert isinstance(str_mlm_method, str) assert isinstance(int_seed, int) or int_seed is None assert str_initial_method_bo in constants.ALLOWED_INITIALIZATIONS_BO assert str_mlm_method in constants.ALLOWED_MLM_METHOD logger.info('arr_range:\n{}'.format( utils_logger.get_str_array(model_bo.arr_range))) logger.info('str_cov: {}'.format(model_bo.str_cov)) logger.info('str_acq: {}'.format(model_bo.str_acq)) logger.info('str_optimizer_method_gp: {}'.format( model_bo.str_optimizer_method_gp)) logger.info('str_optimizer_method_bo: {}'.format( model_bo.str_optimizer_method_bo)) logger.info('str_modelselection_method: {}'.format( model_bo.str_modelselection_method)) logger.info('int_init: {}'.format(int_init)) logger.info('int_iter: {}'.format(int_iter)) logger.info('str_initial_method_bo: {}'.format(str_initial_method_bo)) logger.info('str_initial_method_ao: {}'.format(str_initial_method_ao)) logger.info('int_samples_ao: {}'.format(int_samples_ao)) logger.info('str_mlm_method: {}'.format(str_mlm_method)) logger.info('int_seed: {}'.format(int_seed)) time_start = time.time() X_init = model_bo.get_initial(str_initial_method_bo, fun_objective=fun_target, int_samples=int_init, int_seed=int_seed) if model_bo.debug: logger.debug('X_init:\n{}'.format(utils_logger.get_str_array(X_init))) X_final, Y_final, time_all_final, time_gp_final, time_acq_final = optimize_many( model_bo, fun_target, X_init, int_iter, str_initial_method_ao=str_initial_method_ao, int_samples_ao=int_samples_ao, str_mlm_method=str_mlm_method) time_end = time.time() if model_bo.debug: logger.debug( 'overall time consumed including initializations: {:.4f} sec.'. format(time_end - time_start)) return X_final, Y_final, time_all_final, time_gp_final, time_acq_final