class BayesianOptimizer(object):
def __init__(self, run_multi_threaded=False, metric=None):
if metric is not None:
global MMetric
MMetric = metric
if run_multi_threaded:
self.num_threads = multiprocessing.cpu_count()
else:
self.num_threads = 1
self.pool = None
self.evaluation_depth = maximum_search_points * self.num_threads
self.random_points = 0.0
self.decay = 0.95
if __USE_CPP_BACKEND__:
self.regressor = None
else:
if __REGRESSOR_LIB__ == "GPy":
self.regressor = None
else:
kernel = (ConstantKernel(1.0, constant_value_bounds="fixed") *
sRBF(1.0, length_scale_bounds="fixed"))
self.regressor = GaussianProcessRegressor(copy_X_train=False,
normalize_y=True,
kernel=kernel)
def set_map(self, space_map):
self.regressor = BayesOptimizer_wrapper.PyBayesOptimizer(
extract_map_from_config(space_map))
def reset(self):
del self.regressor
self.regressor = None
self.regressor = GaussianProcessRegressor(copy_X_train=False,
normalize_y=True)
def fit(self, xs, ys):
norm = np.linalg.norm(ys)
ys = ys / norm
if __USE_CPP_BACKEND__:
self.regressor.fit(xs, ys)
else:
ys = np.reshape(ys, (len(ys), 1))
if __REGRESSOR_LIB__ == "GPy":
kern = GPy.kern.RBF(input_dim=1, ARD=True)
M = 15
RR = np.linspace(-1, 1, M)[:, None]
α = 0.0001
self.regressor = GPy.models.SparseGPRegression(xs,
ys,
kern.copy(),
Z=RR.copy())
self.regressor.inference_method = GPy.inference.latent_function_inference.PEP(
α)
self.regressor.optimize(messages=False)
else:
self.regressor.fit(xs, ys)
def create_calc_score_threads(self, sliced_test_points, batch_size):
start = timeit.default_timer()
with Manager() as manager:
threads = []
maxes = []
for i in range(self.num_threads):
maximums = manager.dict()
thread = RegressorThread(regressor=copy.copy(self.regressor),
search_space=sliced_test_points[i],
id=i,
maximums=maximums,
batch_size=batch_size)
thread.start()
threads.append(thread)
maxes.append(maximums)
for thread in threads:
thread.join()
thread.close()
del thread
local_maximums = {}
minimum = 1e9
for i in range(len(maxes)):
for key, value in maxes[i].items():
if len(local_maximums) < batch_size:
local_maximums[key +
i * len(sliced_test_points[-1])] = value
minimum = min(minimum, value)
elif value > minimum:
local_maximums[key +
i * len(sliced_test_points[-1])] = value
for key_2, value_2 in local_maximums.items():
if value_2 == minimum:
local_maximums[key_2]
minimum = min(local_maximums.keys(),
key=(lambda k: local_maximums[k]))
print(timeit.default_timer() - start)
return local_maximums
def predict(self, xs, return_std=False):
if isinstance(self.regressor, "GPy.models.SparseGPRegression"):
mean, var = self.regressor.predict(xs)
if not return_std:
return mean
return mean, var
predicates = self.regressor.predict(xs, return_std=return_std)
return predicates
def next_batch(self, visited, visited_indexes, batch_size, test_points,
visited_results):
if __measure_time__:
start = timeit.default_timer()
if __USE_CPP_BACKEND__:
maximums = self.regressor.next_batch(batch_size, visited_indexes)
if __measure_time__:
stop = timeit.default_timer()
print('Time-find maximums: ', stop - start)
print(
"Visited points so far {}, points to be tested {}".format(
len(visited_indexes), len(test_points)))
return maximums
local_maximums = []
for i in range(len(test_points) // (self.evaluation_depth) + 1):
multi_threaded = False
if self.num_threads > 1:
if min(self.num_threads, 2**21 / len(test_points)) > 1:
multi_threaded = True
self.num_threads = int(
min(self.num_threads,
maximum_search_points / len(test_points)))
if multi_threaded:
step = min(self.evaluation_depth,
len(test_points[i * self.evaluation_depth:])
) // self.num_threads
sliced_test_points = [
test_points[i * self.evaluation_depth +
step * j:i * self.evaluation_depth + step *
(j + 1)] for j in range(self.num_threads)
]
scores = self.create_calc_score_threads(
sliced_test_points, batch_size)
for key, value in scores.items():
local_maximums.append(
[value, key + i * self.evaluation_depth])
else:
end_point = min(len(test_points),
(i + 1) * self.evaluation_depth)
sliced_test_points = test_points[i * (
self.evaluation_depth):end_point]
scores = self.acquisition(
sliced_test_points,
visited_results / np.linalg.norm(visited_results))
accepted = 0
while accepted < batch_size:
arg_max = np.argmax(scores)
if arg_max + (
i *
(self.evaluation_depth)) not in visited_indexes:
local_maximums.append([
scores[arg_max],
arg_max + (i * (self.evaluation_depth))
])
accepted += 1
scores[arg_max] = -1
maximums = []
for _ in range(batch_size):
arg_max = np.argmax(local_maximums, axis=0)
maximums.append(local_maximums[arg_max[0]][1])
local_maximums[arg_max[0]][0] = 0
if __measure_time__:
stop = timeit.default_timer()
print('Time-find maximas: ', stop - start)
print("Visited points so far {}, points to be tested {}".format(
len(visited_indexes), len(test_points)))
for i in range(int(self.random_points * len(maximums))):
x = np.random.randint(0, len(test_points))
while x in visited_indexes or x in maximums:
x = np.random.randint(0, len(test_points))
maximums[-i] = x
self.random_points *= self.decay
return maximums
def acquisition(self, test_points, evaluation_results):
if __ACQUISITION__ == 'PI':
return self.expected_improvement(test_points, evaluation_results)
def PI(self, test_points, evaluation_results):
minimum = min(evaluation_results)
mu, std = self.surrogate(test_points)
mu = mu[:, 0]
probs = norm.cdf(minimum, loc=mu, scale=std)
return probs
def expected_improvement(self, test_points, evaluation_results, xi=0.02):
best = min(evaluation_results)
mu, std = self.surrogate(test_points)
mu = mu[:, 0]
# std = std
with catch_warnings():
# ignore generated warnings
simplefilter("ignore")
imp = mu - best - xi
Z = imp / std
ei = imp * norm.cdf(Z) + std * norm.pdf(Z)
ei[std == 0.0] = 0.0
return ei
def surrogate(self, x):
with catch_warnings():
# ignore generated warnings
simplefilter("ignore")
return self.regressor.predict(x, return_std=True)
