def test_domain():
params = {
'a': range(10),
'b': np.random.randint(0, 1e6, size=50),
'd': [1, 0.1, 2.0],
'e': ['a', 1, 'b', 'c', None],
'f': ['1', '-1'],
'h': [True, False]
}
ds = domain_space(params, domain_size=1000)
assert all(k in ds.mapping_int for k in ['a', 'b'])
assert all(k in ds.mapping_categorical for k in ['d', 'e', 'f', 'h'])
samples = ds.get_domain()
for sample in samples:
for param in params.keys():
assert (sample[param] in params[param])
params = {
'a': [1],
}
ds = domain_space(params, domain_size=1000)
assert all(k in ds.mapping_int for k in ['a'])
samples = ds.get_domain()
for sample in samples:
for param in params.keys():
assert (sample[param] in params[param])
def test_domain():
ds = domain_space(param_dict, domain_size)
# getting the samples from the domain
domain_list = ds.get_domain()
# number of samples should have same size as the domain_size
assert len(domain_list) == domain_size
# change this into the GP domain space
domain_np = ds.convert_GP_space(domain_list)
# in gaussian space verifying correctness of samples sizes and structure
assert domain_np.shape[0] == domain_size
# test the reverse conversion
domain_ps = ds.convert_PS_space(domain_np)
# domain_ps and domain_list should be same
assert type(domain_list) == type(domain_ps)
assert len(domain_list) == len(domain_ps)
# testing samples should be from param_dict and structure is preserved in transformations
if len(domain_list) > 0:
l1 = domain_list[0]
l2 = domain_ps[0]
assert type(l1) == type(l2)
assert len(l1.keys()) == len(l2.keys())
# all keys should be drawn from the param_dict
assert len(l1.keys()) == len(param_dict.keys())
for key in l1.keys():
assert key in param_dict.keys()
ps = dict(x=range(1, 100), y=['a', 'b'], z=uniform(-10, 20))
ds = domain_space(ps, 100)
x = ds.get_domain()
x_gp = ds.convert_GP_space(x)
x_rebuilt = ds.convert_PS_space(x_gp)
for x1, x2 in zip(x, x_rebuilt):
for k in x1.keys():
v1 = x1[k]
v2 = x2[k]
if isinstance(v1, np.float64):
assert v1 == approx(v2, abs=1e-5)
else:
if not v1 == v2:
print(k)
print(x)
print(x_gp)
print(x_rebuilt)
assert v1 == v2
def test_domain():
ds = domain_space(param_dict,domain_size)
#getting the samples from the domain
domain_list = ds.get_domain()
#number of samples should have same size as the domain_size
assert len(domain_list) == domain_size
#change this into the GP domain space
domain_np = ds.convert_GP_space(domain_list)
#in gaussian space verifying correctness of samples sizes and structure
assert domain_np.shape[0] == domain_size
#test the reverse conversion
domain_ps = ds.convert_PS_space(domain_np)
#domain_ps and domain_list should be same
assert type(domain_list) == type(domain_ps)
assert len(domain_list) == len(domain_ps)
#testing samples should be from param_dict and structure is preserved in transformations
if len(domain_list)>0:
l1 = domain_list[0]
l2 = domain_ps[0]
assert type(l1) == type(l2)
assert len(l1.keys()) == len(l2.keys())
#all keys should be drawn from the param_dict
assert len(l1.keys()) == len(param_dict.keys())
for key in l1.keys():
assert key in param_dict.keys()
def runRandomOptimizer(self):
results = dict()
# domain space abstraction
ds = domain_space(self.conf_Dict['param_dict'], self.conf_Dict['domain_size'])
X_sample_list = []
Y_sample_list = []
# running the iterations
for i in range(self.conf_Dict['num_iteration']):
# getting batch by batch random values to try
random_hyper_parameters = ds.get_random_sample(self.conf_Dict['batch_size'])
X_list, Y_list = self.runUserObjective(random_hyper_parameters)
X_sample_list = X_sample_list + X_list
Y_sample_list = Y_sample_list + Y_list
# After all the iterations are done now bookeeping and best hyper parameter values
results['params_tried'] = X_sample_list
results['objective_values'] = Y_sample_list
if len(Y_sample_list) > 0:
results['best_objective'] = np.max(np.array(Y_sample_list))
results['best_params'] = X_sample_list[np.argmax(np.array(Y_sample_list))]
return results
def test_gp_samples_to_params():
space = {
'a': range(10),
'b': uniform(-10, 20),
'c': ['cat1', 1, 'cat2'],
'e': [1, 2, 3],
'f': ['const'],
'g': loguniform(0.001, 100),
'h': [10]
}
X = np.array([
# 4, -8, 'cat2', 1, 'const', 1 , 10
[0.4444, 0.1, 0, 0, 1, 0, 1, 0.6, 0],
# 0, -10.0, 'cat1', 3, 'const', 0.001 , 10
[0.0, 0.0, 1, 0, 0, 1, 1, 0.0, 0],
# 9, 10.0, 1, 2, 'const', 100 , 10
[1.0, 1.0, 0, 1, 0, 0.5, 1, 1.0, 0],
])
expected = [
dict(a=4, b=-8.0, c='cat2', e=1, f='const', g=1, h=10),
dict(a=0, b=-10.0, c='cat1', e=3, f='const', g=.001, h=10),
dict(a=9, b=10.0, c=1, e=2, f='const', g=100, h=10),
]
ds = domain_space(space, domain_size=1000)
params = ds.convert_to_params(X)
for act, exp in zip(params, expected):
for k, v in act.items():
if k == 'g':
assert np.isclose(v, exp[k])
else:
assert v == exp[k]
def runRandomOptimizer(self):
results = dict()
# domain space abstraction
ds = domain_space(self.param_dict, self.config.domain_size)
X_sample_list = []
Y_sample_list = []
# running the iterations
pbar = tqdm(range(self.config.num_iteration))
for i in pbar:
# getting batch by batch random values to try
random_hyper_parameters = ds.get_random_sample(self.config.batch_size)
X_list, Y_list = self.runUserObjective(random_hyper_parameters)
X_sample_list = np.append(X_sample_list, X_list)
Y_sample_list = np.append(Y_sample_list, Y_list)
pbar.set_description("Best score: %s" % np.max(np.array(Y_sample_list)))
# After all the iterations are done now bookkeeping and best hyper parameter values
results['params_tried'] = X_sample_list
results['objective_values'] = Y_sample_list
if len(Y_sample_list) > 0:
results['best_objective'] = np.max(Y_sample_list)
results['best_params'] = X_sample_list[np.argmax(Y_sample_list)]
if self.maximize_objective is False:
results['objective_values'] = -1 * results['objective_values']
results['best_objective'] = -1 * results['best_objective']
return results
def test_mango_loguniform():
space = {
'a': mango_loguniform(-3, 6)
}
ds = domain_space(space, domain_size=1000)
samples = ds.get_domain()
assert all(1e-3 < sample['a'] < 1e3 for sample in samples)
def test_np_space():
space = {
'x': np.array(['a', 'b', 'c']),
'y': np.arange(100),
}
ds = domain_space(space, domain_size=10)
params = ds.get_domain()
assert len(params) == 10
gp_params = ds.convert_GP_space(params)
assert gp_params.shape == (10, 4)
def test_domain():
params = {
'a': (1, 2, 4),
'b': np.random.randint(0, 1e6, size=50),
'c': np.random.uniform(low=-100., high=100., size=(50, ))
}
print(params)
ds = domain_space(params, domain_size=100)
samples = ds.get_domain()
for sample in samples:
for param in params.keys():
assert (sample[param] in params[param])
def test_gp_space():
space = {
'f': range(10),
'h': uniform(-10, 20),
'e': ['cat1', 1, 'cat2'],
'c': [1, 2, 3],
'a': ['const'],
'g': loguniform(0.001, 100),
'b': [10],
'd': uniform(0, 1),
'i': [True, False]
}
ds = domain_space(space, domain_size=10000)
X = ds.sample_gp_space()
assert (X <= 1.0).all()
assert (X >= 0.0).all()
assert (X[:, 0] == 1.).all() # a
assert (X[:, 1] == 0.).all() # b
assert np.isin(X[:, 2], [0.0, 0.5, 1.0]).all() # c
assert np.isin(X[:, 4:7], np.eye(3)).all() # e
assert X.shape == (ds.domain_size, 12)
params = ds.convert_to_params(X)
for param in params:
assert param['a'] == 'const'
assert param['b'] == 10
assert param['c'] in space['c']
assert 0.0 <= param['d'] <= 1.0
assert param['e'] in space['e']
assert param['f'] in space['f']
assert 0.001 <= param['g'] <= 100
assert -10 <= param['h'] <= 10
assert param['i'] in space['i']
X2 = ds.convert_to_gp(params)
assert np.isclose(X2, X).all()
def __init__(self, param_dict, objective, conf_dict=None):
self.param_dict = param_dict
self.objective_function = objective
self.maximize_objective = True
if conf_dict is None:
conf_dict = {}
self.config = Tuner.Config(**conf_dict)
if self.config.domain_size is None:
self.config.domain_size = self.calculateDomainSize(self.param_dict)
# overwrite batch size if given as a property of objective function
if hasattr(objective, 'batch_size'):
self.config.batch_size = objective.batch_size
# save domain size
self.ds = domain_space(self.param_dict, self.config.domain_size)
# stores the results of using the tuner
self.results = dict()
def runBayesianOptimizer(self):
results = dict()
# domain space abstraction
ds = domain_space(self.conf_Dict['param_dict'], self.conf_Dict['domain_size'])
# getting first few random values
random_hyper_parameters = ds.get_random_sample(self.conf_Dict['initial_random'])
X_list, Y_list = self.runUserObjective(random_hyper_parameters)
# in case initial random results are invalid try different samples
n_tries = 1
while len(Y_list) < self.conf_Dict['initial_random'] and n_tries < 3:
random_hps = ds.get_random_sample(self.conf_Dict['initial_random'] - len(Y_list))
X_list2, Y_list2 = self.runUserObjective(random_hps)
random_hyper_parameters.extend(random_hps)
X_list.extend(X_list2)
Y_list.extend(Y_list2)
n_tries += 1
if len(Y_list) == 0:
raise ValueError("No valid configuration found to initiate the Bayesian Optimizer")
# evaluated hyper parameters are used
X_init = ds.convert_GP_space(X_list)
Y_init = np.array(Y_list).reshape(len(Y_list), 1)
# setting the initial random hyper parameters tried
results['random_params'] = X_list
results['random_params_objective'] = Y_list
Optimizer = BayesianLearning(surrogate=self.conf_Dict.get('surrogate'))
Optimizer.domain_size = self.conf_Dict['domain_size']
X_sample = X_init
Y_sample = Y_init
hyper_parameters_tried = random_hyper_parameters
objective_function_values = Y_list
# running the iterations
for i in range(self.conf_Dict['num_iteration']):
# Domain Space
domain_list = ds.get_domain()
X_domain_np = ds.convert_GP_space(domain_list)
# Black-Box Optimizer
X_next_batch = Optimizer.get_next_batch(X_sample, Y_sample, X_domain_np,
batch_size=self.conf_Dict['batch_size'])
# X_next_batch = Optimizer.get_next_batch_clustering(X_sample,Y_sample,X_domain_np,batch_size=self.conf_Dict['batch_size'])
# Scheduler
X_next_PS = ds.convert_PS_space(X_next_batch)
# Evaluate the Objective function
# Y_next_batch, Y_next_list = self.runUserObjective(X_next_PS)
X_next_list, Y_next_list = self.runUserObjective(X_next_PS)
Y_next_batch = np.array(Y_next_list).reshape(len(Y_next_list), 1)
# update X_next_batch to successfully evaluated values
X_next_batch = ds.convert_GP_space(X_next_list)
# update the bookeeping of values tried
hyper_parameters_tried = hyper_parameters_tried + X_next_list
objective_function_values = objective_function_values + Y_next_list
# Appending to the current samples
X_sample = np.vstack((X_sample, X_next_batch))
Y_sample = np.vstack((Y_sample, Y_next_batch))
results['params_tried'] = hyper_parameters_tried
results['objective_values'] = objective_function_values
results['best_objective'] = np.max(Y_sample)
results['best_params'] = hyper_parameters_tried[np.argmax(Y_sample)]
# saving the optimizer and ds in the tuner object which can save the surrogate function and ds details
self.Optimizer = Optimizer
self.ds = ds
return results
def runExponentialTuner(self):
"""
Steps:
1-Create DS obj for each obj of param_dict_list
2-Sample randomly from the DS objects and evaluate the objective functions.
3- Now use the GPR for each objective to select next batch
4- Select the best values based on fxn(surrogate) from GPR.
5- Modify the surrogate selections so as to avoid getting struck.
"""
results = dict()
#random params tried
results['random_params'] = []
#random objective values
results['random_params_objective'] = []
#random objective function id
results['random_objective_fid'] = []
results['params_tried'] = []
results['objective_values'] = []
results['objective_fid'] = []
#print('*** Entering Metatuner *** ')
num_of_random = self.initial_random
ds = []
for i in self.param_dict_list:
domain_size = self.calculateDomainSize(i)
ds.append(domain_space(i, domain_size))
#dict of list for each obj function
X_dict_list = {}
Y_dict_list = {}
#dict of array for each obj function
X_dict_array = {}
Y_dict_array = {}
#stores the maximum value of the objective function for each objective
Y_dict_array_max = {}
#randomly evaluate the initial points for objectives
for i in range(len(self.param_dict_list)):
#used later
self.first_itr[i] = True
if self.debug:
#fixing random seeds to reproduce the results for debugging
np.random.seed(self.seed)
random.seed(self.seed)
ds_i = ds[i]
#make sure not all values in y are same
done = False
#count no of times random repeated
num_times = 0
while not done:
random_hyper_parameters = ds_i.get_random_sample(num_of_random)
y_list = self.objective_list[i](random_hyper_parameters)
#print(i, random_hyper_parameters, y_list)
num_times = num_times + 1
#10 times done still there is a tie
if num_times == 10:
done = True
y_list[0] = y_list[0] - 0.001
for ind in range(num_of_random):
if y_list[0] != y_list[ind]:
done = True
#print('Random_hyper_parameters:',random_hyper_parameters, y_list)
x_list = random_hyper_parameters
#print(i, random_hyper_parameters, x_list, y_list)
X_dict_list[i] = []
X_dict_list[i].append(x_list)
Y_dict_list[i] = []
Y_dict_list[i].append(y_list)
self.objective_values_list += y_list
results['random_params'] = results['random_params'] + x_list
results['random_params_objective'] = results[
'random_params_objective'] + y_list
results['random_objective_fid'] = results[
'random_objective_fid'] + [i] * len(x_list)
#x_array2 = ds_i.convert_GP_space(random_hyper_parameters)
#x_array = ds_i.convert_to_gp(random_hyper_parameters)
x_array = ds_i.convert_GP_space(random_hyper_parameters)
X_dict_array[i] = x_array
#print(x_array2)
#print(x_array)
y_array = np.array(y_list).reshape(len(y_list), 1)
Y_dict_array[i] = y_array
#the random ones are added as it is
Y_dict_array_max[i] = y_array
#Initialize the number of Optimizers
Optimizer_list = []
for i in range(len(self.objective_list)):
Optimizer_i = BayesianLearning(surrogate=self.surrogate)
Optimizer_i.domain_size = ds[i].domain_size
Optimizer_list.append(Optimizer_i)
#print('Domain Size:',ds[i].domain_size)
# Storing optimizers attributes: iteration count and modified exploration of external surrogate
Optimizer_exploration = []
Optimizer_iteration = []
for i in range(len(self.objective_list)):
Optimizer_exploration.append(1.0)
Optimizer_iteration.append(1.0)
results['params_tried'] = results['random_params']
results['objective_values'] = results['random_params_objective']
results['objective_fid'] = results['random_objective_fid']
#print(Optimizer_exploration)
#Now run the optimization iterations
pbar = tqdm(range(self.num_of_iterations))
for itr in pbar:
#next values of x returned from individual function
#Values in x are dependent on types of param dict, so using a list
x_values_list = []
#Next promising surrogate values for each objective: used to select functions
s_values_array = np.empty((0, 1), float)
#keeping track of objective indices: Maps the x values to its objective function
x_obj_indices = []
#Next promising surrogate values for each objective in list form
s_values_list = []
#sample individual domains and evaluate surrogate functions.
#we get the next promising samples along with the surrogate function values
max_val_y = self.get_max_y_value(Y_dict_array)
#In GPs this value is used to scale the Y values
max_val_y_scaled = max_val_y
#In this setting the default GP performs reasonably well
if max_val_y_scaled < 1.0:
max_val_y_scaled = 1.0
for j in range(len(ds)):
if self.debug:
#fixing random seeds to reproduce the results for debugging
np.random.seed(self.seed)
random.seed(self.seed)
#X_domain_np = ds[j].sample_gp_space()
if self.first_itr[j]:
#if j==self.last_used or self.first_itr[j]:
# Domain Space
domain_list = ds[j].get_domain()
self.store_domain[j] = ds[j].convert_GP_space(domain_list)
#self.store_domain[j] = ds[j].sample_gp_space()
self.first_itr[j] = True
#print('doing domain sampling')
X_domain_np = self.store_domain[j]
#next batch of x for this objective along with its surrogate value
X_next_batch, surr_value, surr_value_ext, u_values = Optimizer_list[
j].get_next_batch_MetaTuner(
X_dict_array[j],
Y_dict_array[j] / max_val_y_scaled,
X_domain_np,
self.obj_batch_size,
Optimizer_exploration[j],
Optimizer_iteration[j],
classifier_index=j,
last_used_index=j) #self.last_used
#used for displaying and debugging
#s_values_list.append([int(surr_value_ext[0]*100.0),int(surr_value[0]*100.0)])
s_values_list.append([int(surr_value_ext[0] * 100.0)])
#this is used to sort the objective values
s_value = np.array(surr_value_ext) #modified sandeep
#s_value = np.array(surr_value_ext)
s_value = s_value.reshape(-1, s_value.shape[0])
s_values_array = np.append(s_values_array, s_value)
x_values_list = x_values_list + X_next_batch
#keep track of objective function for corresponding surrogate and x values
for k in range(self.obj_batch_size):
x_obj_indices.append(j)
#sort the surrogate values in descending order, to select the best value from them
v_sorting_index = np.argsort(-s_values_array, axis=0)
#now select the self.batch_size values from x_values_list based on v_sorting_index
v_sorting_index = v_sorting_index[:self.batch_size]
#select randomly with exploration rate else select the max surrogate
prob_selection = random.random()
random_selected = False
#do the random evaluation of the functions
if prob_selection < self.exploration_rate:
selected_obj = random.randint(
0, (len(ds) - 1)) #0 and len(ds included)
if self.exploration_rate > self.exploration_rate_min:
self.exploration_rate = self.exploration_rate * self.decay_rate
else:
self.exploration_rate = self.exploration_rate_min
random_selected = True
else:
selected_obj = v_sorting_index[0]
#keep track of objective indices selected in current iteration
loc_indices = []
curr_x_next_np = x_values_list[selected_obj]
#convert this into the parameter space for scheduling
#see the function index for this x value
index = selected_obj
# updating last used classifier
self.last_used = index
#keep track of local indices
loc_indices.append(index)
#keep track of indices in a global datastr for visualization of function selection
self.objectives_evaluated.append(index)
curr_x_next = ds[index].convert_PS_space(curr_x_next_np)
#run the next curr_x_next value for the objective function
y_list = self.objective_list[index](curr_x_next)
self.objective_values_list += y_list
results['params_tried'] = results['params_tried'] + curr_x_next
results['objective_values'] = results['objective_values'] + y_list
results['objective_fid'] = results['objective_fid'] + [index]
curr_y_array = np.array(y_list).reshape(len(y_list), 1)
#append the curr_x_next_np, curr_x_next, y_list to appropriate datastructures for book keeping
X_dict_array[index] = np.vstack(
(X_dict_array[index], curr_x_next_np))
Y_dict_array[index] = np.vstack(
(Y_dict_array[index], curr_y_array))
Y_dict_array_max[index] = np.vstack(
(Y_dict_array_max[index], np.max(Y_dict_array[index])))
#scale the exploration of objectives that are not selected for others make their exploration to 1
for i in range(len(self.objective_list)):
if i in loc_indices:
Optimizer_exploration[i] = 1.0
Optimizer_iteration[i] += 1.0
else:
Optimizer_exploration[i] = Optimizer_exploration[
i] + 1.1 * self.exploration_rate
pbar.set_description(": Best score: %s" % max_val_y)
#print(itr, s_values_list, Optimizer_iteration, Optimizer_exploration, max_val_y)#, Y_dict_array[selected_obj])
#print(itr, s_values_list, random_selected, self.exploration_rate, prob_selection, Optimizer_iteration , max_val_y)
#print(itr, s_values_list, random_selected, Optimizer_iteration , int(self.exploration_rate*100.0), max_val_y)
self.X_dict_array = X_dict_array
self.Y_dict_array = Y_dict_array
self.Y_dict_array_max = Y_dict_array_max
self.ds = ds
self.X_dict_array = X_dict_array
self.Y_dict_array = Y_dict_array
self.Y_dict_array_max = Y_dict_array_max
self.ds = ds
index_best = np.argmax(results['objective_values'])
results['best_objective'] = results['objective_values'][index_best]
results['best_params'] = results['params_tried'][index_best]
results['best_objective_fid'] = results['objective_fid'][index_best]
return results
def runBayesianOptimizer(self):
results = dict()
# domain space abstraction
ds = domain_space(self.param_dict, self.config.domain_size)
# getting first few random values
random_hyper_parameters = ds.get_random_sample(self.config.initial_random)
X_list, Y_list = self.runUserObjective(random_hyper_parameters)
# in case initial random results are invalid try different samples
n_tries = 1
while len(Y_list) < self.config.initial_random and n_tries < 3:
random_hps = ds.get_random_sample(self.config.initial_random - len(Y_list))
X_list2, Y_list2 = self.runUserObjective(random_hps)
random_hyper_parameters.extend(random_hps)
X_list = np.append(X_list, X_list2)
Y_list = np.append(Y_list, Y_list2)
n_tries += 1
if len(Y_list) == 0:
raise ValueError("No valid configuration found to initiate the Bayesian Optimizer")
# evaluated hyper parameters are used
X_init = ds.convert_to_gp(X_list)
Y_init = Y_list.reshape(len(Y_list), 1)
# setting the initial random hyper parameters tried
results['random_params'] = X_list
results['random_params_objective'] = Y_list
Optimizer = BayesianLearning(surrogate=self.config.surrogate, n_features=X_init.shape[1])
Optimizer.domain_size = self.config.domain_size
X_sample = X_init
Y_sample = Y_init
hyper_parameters_tried = random_hyper_parameters
objective_function_values = Y_list
surrogate_values = Y_list
x_failed_evaluations = np.array([])
# running the iterations
pbar = tqdm(range(self.config.num_iteration))
for i in pbar:
# Domain Space
X_domain_np = ds.sample_gp_space()
# Black-Box Optimizer
if self.config.scale_y:
scale = np.max(Y_sample) - np.min(Y_sample)
if scale == 0:
scale = np.max(np.abs(Y_sample))
Y_scaled = (Y_sample - np.mean(Y_sample)) / scale
else:
Y_scaled = Y_sample
if self.config.strategy_is_penalty:
X_next_batch = Optimizer.get_next_batch(X_sample, Y_scaled, X_domain_np,
batch_size=self.config.batch_size)
elif self.config.strategy_is_clustering:
X_next_batch = Optimizer.get_next_batch_clustering(X_sample,Y_scaled, X_domain_np,
batch_size=self.config.batch_size)
else:
# assume penalty approach
X_next_batch = Optimizer.get_next_batch(X_sample, Y_scaled, X_domain_np,
batch_size=self.config.batch_size)
# Scheduler
X_next_PS = ds.convert_to_params(X_next_batch)
# if all the xs have failed before, replace them with random sample
# as we will not get any new information otherwise
if all(x in x_failed_evaluations for x in X_next_PS):
X_next_PS = ds.get_random_sample(self.config.batch_size)
# Evaluate the Objective function
# Y_next_batch, Y_next_list = self.runUserObjective(X_next_PS)
X_next_list, Y_next_list = self.runUserObjective(X_next_PS)
# keep track of all parameters that failed
x_failed = [x for x in X_next_PS if x not in X_next_list]
x_failed_evaluations = np.append(x_failed_evaluations, x_failed)
if len(Y_next_list) == 0:
# no values returned
# this is problematic if domain is small and same value is tried again in the next iteration as the optimizer would be stuck
continue
Y_next_batch = Y_next_list.reshape(len(Y_next_list), 1)
# update X_next_batch to successfully evaluated values
X_next_batch = ds.convert_to_gp(X_next_list)
# update the bookeeping of values tried
hyper_parameters_tried = np.append(hyper_parameters_tried, X_next_list)
objective_function_values = np.append(objective_function_values, Y_next_list)
surrogate_values = np.append(surrogate_values, Optimizer.surrogate.predict(X_next_batch))
# Appending to the current samples
X_sample = np.vstack((X_sample, X_next_batch))
Y_sample = np.vstack((Y_sample, Y_next_batch))
pbar.set_description("Best score: %s" % np.max(Y_sample))
results['params_tried'] = hyper_parameters_tried
results['objective_values'] = objective_function_values
results['surrogate_values'] = surrogate_values
results['best_objective'] = np.max(Y_sample)
results['best_params'] = hyper_parameters_tried[np.argmax(Y_sample)]
if self.maximize_objective is False:
results['objective_values'] = -1 * results['objective_values']
results['best_objective'] = -1 * results['best_objective']
# saving the optimizer and ds in the tuner object which can save the surrogate function and ds details
self.Optimizer = Optimizer
self.ds = ds
return results
