def training_xgb_model3(X_train, y_train, seed=37):
def treesCV(eta, gamma, max_depth, min_child_weight, subsample,
colsample_bytree, n_estimators):
#function for cross validation gradient boosted trees
return cross_val_score(xgb.XGBRegressor(
objective='binary:logistic',
tree_method='hist',
learning_rate=max(eta, 0),
gamma=max(gamma, 0),
max_depth=int(max_depth),
min_child_weight=int(min_child_weight),
silent=True,
subsample=max(min(subsample, 1), 0.0001),
colsample_bytree=max(min(colsample_bytree, 1), 0.0001),
n_estimators=int(n_estimators),
seed=42,
nthread=-1),
X=X_train,
y=y_train,
cv=cv_splits,
n_jobs=-1).mean()
#Bayesian Hyper parameter optimization of gradient boosted trees
treesBO = BayesianOptimization(
treesCV, {
'eta': (0.001, 0.4),
'gamma': (8, 12),
'max_depth': (400, 700),
'min_child_weight': (0.1, 1),
'subsample': (0.3, 0.6),
'colsample_bytree': (0.6, 1),
'n_estimators': (600, 800)
})
treesBO.maximize(n_iter=iter_no, **gp_params)
tree_best = treesBO.res['max']
#train tree with best paras
trees_model = xgb.XGBRegressor(
objective='binary:logistic',
tree_method='hist',
seed=42,
learning_rate=max(tree_best['max_params']['eta'], 0),
gamma=max(tree_best['max_params']['gamma'], 0),
max_depth=int(tree_best['max_params']['max_depth']),
min_child_weight=int(tree_best['max_params']['min_child_weight']),
silent=True,
subsample=max(min(tree_best['max_params']['subsample'], 1), 0.0001),
colsample_bytree=max(
min(tree_best['max_params']['colsample_bytree'], 1), 0.0001),
n_estimators=int(tree_best['max_params']['n_estimators']),
nthread=-1)
trees_model.fit(X_train, y_train)
return trees_model
def main():
"""
main function
"""
rospy.init_node("scenario_optimizer", anonymous=True)
scenario_optimizer = ScenarioOptimizer()
pbounds = {"walk_x": (-0.1, 0.1), "walk_y": (0.75, 1.25)}
bounds_transformer = SequentialDomainReductionTransformer()
optimizer = BayesianOptimization(f=scenario_optimizer.run_scenario,
pbounds=pbounds,
verbose=2,
random_state=1,
bounds_transformer=bounds_transformer)
optimizer.probe(params={"walk_x": 0.0, "walk_y": 1.0}, lazy=True)
optimizer.maximize(init_points=10, n_iter=20)
scenario_optimizer.plot_graph()
if scenario_optimizer is not None:
#scenario_optimizer.print_result()
del scenario_optimizer
rospy.loginfo("Done")
def test_get_new_suggestions(self):
bayesianOptimization = BayesianOptimization()
new_trials = bayesianOptimization.get_new_suggestions(
self.study.id, self.trials, 1)
new_trials[0].status = "Completed"
new_trials[0].parameter_values = '{"hidden1": 50}'
new_trials[0].objective_value = 0.6
new_trials[0].save()
new_trials = bayesianOptimization.get_new_suggestions(
self.study.id, self.trials, 1)
new_trials[0].status = "Completed"
new_trials[0].parameter_values = '{"hidden1": 150}'
new_trials[0].objective_value = 0.8
new_trials[0].save()
new_trials = bayesianOptimization.get_new_suggestions(
self.study.id, self.trials, 1)
new_trials[0].status = "Completed"
new_trials[0].parameter_values = '{"hidden1": 250}'
new_trials[0].objective_value = 0.6
new_trials[0].save()
new_trials = bayesianOptimization.get_new_suggestions(
self.study.id, self.trials, 1)
# Assert getting two trials
self.assertEqual(len(new_trials), 1)
# Assert getting the trials
new_trial = new_trials[0]
new_parameter_values = new_trial.parameter_values
new_parameter_values_json = json.loads(new_parameter_values)
from bayesian_optimization import UtilityFunction, BayesianOptimization
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from TwoDimEnvironmentbc import environment_array
utility_function = UtilityFunction(kind="ei", xi=0, kappa=0)
def target(x, y):
return environment_array(x, y)
optimizer = BayesianOptimization(target, {
'y': (2, 4),
'x': (40, 65)
},
random_state=112)
#optimizer.maximize(init_points=int(input('Enter the number of random steps: ')),n_iter=0)
def probe_point(x, y):
return optimizer.probe(
params={
"x": x,
"y": y
},
lazy=True,
)
cv.append(c)
ucv.append(d)
'''Instantiates the bounded region of parameter space NB: Welding energy is scaled down by a factor of 100'''
pbounds = {'y': (40, 65), 'z': (2, 4),'x':(10,40)}
'''optimizer calls the BayesianOptimization class, defines the black box function as function from above and takes
the bounded region of parameter space pbounds from above. random_state is a variable which can be specified to make each
set of random experiments repeatable. The results presented in the report can be obtained by setting random_state = 2
and random_state = 1 respectively'''
optimizer = BayesianOptimization(
f=function,
pbounds=pbounds,
verbose=2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
random_state=2
)
def probe_point(x,y,z):
'''This is a function that allows the user to probe specific points in the environment
and add them to the environment observations this can be used to guide the optimisation process'''
return optimizer.probe(params={"x": x, "y": y, "z":z},lazy=True,)
'''The alpha parameter controls how the environment deals with noise. This can be increased in cases where the environment
is more complex. Its base value is 1e-5'''
alpha = 3e-4
'''The Matern kernel is discussed in section 5.2 of the report. nu is the kernel hyperparameter.'''
kernel = Matern(nu = 2.5)
n_estimators=int(n_estimators)),
train,
train_labels,
'roc_auc',
cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = \
make_data(train_path ="../input/xtrain_v5_full.csv",
test_path="../input/xtest_v5.csv")
# RF
etcBO = BayesianOptimization(etccv, {
'n_estimators': (200, 800),
'min_samples_split': (2, 8)
})
print('-' * 53)
etcBO.maximize()
print('-' * 53)
print('Final Results')
print('ETC: %f' % etcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(ETC(
n_estimators=int(etcBO.res['max']['max_params']['n_estimators']),
min_samples_split=int(
etcBO.res['max']['max_params']['min_samples_split']),
random_state=2,
n_jobs=-1),
train,
def target(c):
return environment_array(c)
b = np.linspace(40, 65, 1000)
c = np.linspace(2, 4, 1000).reshape(-1, 1)
y = target(c)
plt.title('Function to be optimised')
plt.xlabel('Clamping pressure (bar)')
plt.ylabel('Lap shear strength (N)')
plt.plot(c, y)
plt.savefig('Function to be optimised')
plt.show()
optimizer = BayesianOptimization(target, {'c': (2, 4)}, random_state=27)
optimizer.maximize(init_points=2, n_iter=0, kappa=5, xi=0)
def posterior(optimizer, x_obs, y_obs, grid):
optimizer._gp.fit(x_obs, y_obs)
mu, sigma = optimizer._gp.predict(grid, return_std=True)
return mu, sigma
def plot_gp(optimizer, x, y):
fig = plt.figure(figsize=(16, 10))
steps = len(optimizer.space)
fig.suptitle(
return cross_val_score(AdaBoostClassifier(ETC(min_samples_split=int(min_samples_split),
random_state=2,
n_jobs=-1),
algorithm="SAMME",
n_estimators=int(n_estimators)),
train, train_labels, 'roc_auc', cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = \
make_data(train_path ="../input/xtrain_v5_full.csv",
test_path="../input/xtest_v5.csv")
# RF
etcBO = BayesianOptimization(etccv, {'n_estimators': (200, 800),
'min_samples_split': (2, 8)})
print('-' * 53)
etcBO.maximize()
print('-' * 53)
print('Final Results')
print('ETC: %f' % etcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(ETC(n_estimators=int(etcBO.res['max']['max_params']['n_estimators']),
min_samples_split=int(etcBO.res['max']['max_params']['min_samples_split']),
random_state=2,
n_jobs=-1),
train, train_labels, 'roc_auc', cv=5).mean()
rf.fit(train, train_labels)
preds = rf.predict_proba(test)[:, 1]
regressor = SVR(kernel='rbf')
regressor.fit(X, Y)
utility_function = UtilityFunction(kind="ei", xi=0, kappa=0)
def target(x, y, z):
return float(
sc_Y.inverse_transform(
regressor.predict(
sc_X.transform(np.array([x, y, z]).reshape(1, -1)))))
optimizer = BayesianOptimization(target, {
'z': (2, 4),
'y': (40, 65),
'x': (1000, 4000)
},
random_state=250)
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
a = []
b = []
j = []
cv = []
def utilitytarget(xtarget, ytarget, ztarget):
xyparam = np.array([[xtarget, ytarget, ztarget]])
return float((utility_function.utility(xyparam, optimizer._gp, 0)))
for t, x, y in data(train, D): # data is a generator
p = learner.predict(x)
learner.update(x, p, y)
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = make_data(train_path = "../input/train.csv", test_path="../input/test.csv")
ftrlBO = BayesianOptimization(xgboostcv,
{'max_depth': (7, 20),
'learning_rate': (0.45, 0.01),
'n_estimators': (100, 500),
'gamma': (1., 0.1),
'min_child_weight': (2, 15),
'max_delta_step': (0.6, 0.4),
'subsample': (0.7, 0.9),
'colsample_bytree': (0.7, 0.9)
})
ftrlBO.maximize(init_points=5, restarts=50, n_iter=25)
print('-' * 53)
print('Final Results')
print('FTRL: %f' % ftrlBO.res['max']['max_val'])
# Build and Run on the full data set and the validation set for ensembling later.
clf = XGBClassifier(max_depth=int(xgboostBO.res['max']['max_params']['max_depth']),
def test_init(self): bayesianOptimization = BayesianOptimization() self.assertEqual(bayesianOptimization.__class__, BayesianOptimization)
def search(self,
dictionary,
metric="MAP",
init_points=5,
n_cases=30,
output_root_path=None,
parallelPoolSize=2,
parallelize=True,
save_model="best"):
# Associate the params that will be returned by BayesianOpt object to those you want to save
# E.g. with early stopping you know which is the optimal number of epochs only afterwards
# but you might want to save it as well
self.from_fit_params_to_saved_params = {}
self.dictionary_input = dictionary.copy()
hyperparamethers_range_dictionary = dictionary[
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS].copy()
self.output_root_path = output_root_path
self.logFile = open(self.output_root_path + "_BayesianSearch.txt", "a")
self.save_model = save_model
self.model_counter = 0
self.categorical_mapper_dict_case_to_index = {}
self.categorical_mapper_dict_index_to_case = {}
# Transform range element in a list of two elements: min, max
for key in hyperparamethers_range_dictionary.keys():
# Get the extremes for every range
current_range = hyperparamethers_range_dictionary[key]
if type(current_range) is range:
min_val = current_range.start
max_val = current_range.stop
elif type(current_range) is list:
categorical_mapper_dict_case_to_index_current = {}
categorical_mapper_dict_index_to_case_current = {}
for current_single_case in current_range:
num_vaues = len(
categorical_mapper_dict_case_to_index_current)
categorical_mapper_dict_case_to_index_current[
current_single_case] = num_vaues
categorical_mapper_dict_index_to_case_current[
num_vaues] = current_single_case
num_vaues = len(categorical_mapper_dict_case_to_index_current)
min_val = 0
max_val = num_vaues - 1
self.categorical_mapper_dict_case_to_index[
key] = categorical_mapper_dict_case_to_index_current.copy(
)
self.categorical_mapper_dict_index_to_case[
key] = categorical_mapper_dict_index_to_case_current.copy(
)
else:
raise TypeError(
"BayesianSearch: for every parameter a range may be specified either by a 'range' object or by a list."
"Provided object type for parameter '{}' was '{}'".format(
key, type(current_range)))
hyperparamethers_range_dictionary[key] = [min_val, max_val]
self.runSingleCase_partial = partial(self.runSingleCase,
dictionary=dictionary,
metric=metric)
self.bayesian_optimizer = BayesianOptimization(
self.runSingleCase_partial, hyperparamethers_range_dictionary)
self.best_solution_val = None
self.best_solution_parameters = None
#self.best_solution_object = None
self.bayesian_optimizer.maximize(init_points=init_points,
n_iter=n_cases,
kappa=2)
best_solution = self.bayesian_optimizer.res['max']
self.best_solution_val = best_solution["max_val"]
self.best_solution_parameters = best_solution["max_params"].copy()
self.best_solution_parameters = self.parameter_bayesian_to_token(
self.best_solution_parameters)
self.best_solution_parameters = self.from_fit_params_to_saved_params[
frozenset(self.best_solution_parameters.items())]
writeLog(
"BayesianSearch: Best config is: Config {}, {} value is {:.4f}\n".
format(self.best_solution_parameters, metric,
self.best_solution_val), self.logFile)
#
#
# if folderPath != None:
#
# writeLog("BayesianSearch: Saving model in {}\n".format(folderPath), self.logFile)
# self.runSingleCase_param_parsed(dictionary, metric, self.best_solution_parameters, folderPath = folderPath, namePrefix = namePrefix)
return self.best_solution_parameters.copy()
class BayesianSearch(AbstractClassSearch):
ALGORITHM_NAME = "BayesianSearch"
"""
This class applies Bayesian parameter tuning using this package:
https://github.com/fmfn/BayesianOptimization
pip install bayesian-optimization
"""
def __init__(self,
recommender_class,
evaluator_validation=None,
evaluator_test=None):
super(BayesianSearch,
self).__init__(recommender_class,
evaluator_validation=evaluator_validation,
evaluator_test=evaluator_test)
def search(self,
dictionary,
metric="MAP",
init_points=5,
n_cases=30,
output_root_path=None,
parallelPoolSize=2,
parallelize=True,
save_model="best"):
# Associate the params that will be returned by BayesianOpt object to those you want to save
# E.g. with early stopping you know which is the optimal number of epochs only afterwards
# but you might want to save it as well
self.from_fit_params_to_saved_params = {}
self.dictionary_input = dictionary.copy()
hyperparamethers_range_dictionary = dictionary[
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS].copy()
self.output_root_path = output_root_path
self.logFile = open(self.output_root_path + "_BayesianSearch.txt", "a")
self.save_model = save_model
self.model_counter = 0
self.categorical_mapper_dict_case_to_index = {}
self.categorical_mapper_dict_index_to_case = {}
# Transform range element in a list of two elements: min, max
for key in hyperparamethers_range_dictionary.keys():
# Get the extremes for every range
current_range = hyperparamethers_range_dictionary[key]
if type(current_range) is range:
min_val = current_range.start
max_val = current_range.stop
elif type(current_range) is list:
categorical_mapper_dict_case_to_index_current = {}
categorical_mapper_dict_index_to_case_current = {}
for current_single_case in current_range:
num_vaues = len(
categorical_mapper_dict_case_to_index_current)
categorical_mapper_dict_case_to_index_current[
current_single_case] = num_vaues
categorical_mapper_dict_index_to_case_current[
num_vaues] = current_single_case
num_vaues = len(categorical_mapper_dict_case_to_index_current)
min_val = 0
max_val = num_vaues - 1
self.categorical_mapper_dict_case_to_index[
key] = categorical_mapper_dict_case_to_index_current.copy(
)
self.categorical_mapper_dict_index_to_case[
key] = categorical_mapper_dict_index_to_case_current.copy(
)
else:
raise TypeError(
"BayesianSearch: for every parameter a range may be specified either by a 'range' object or by a list."
"Provided object type for parameter '{}' was '{}'".format(
key, type(current_range)))
hyperparamethers_range_dictionary[key] = [min_val, max_val]
self.runSingleCase_partial = partial(self.runSingleCase,
dictionary=dictionary,
metric=metric)
self.bayesian_optimizer = BayesianOptimization(
self.runSingleCase_partial, hyperparamethers_range_dictionary)
self.best_solution_val = None
self.best_solution_parameters = None
#self.best_solution_object = None
self.bayesian_optimizer.maximize(init_points=init_points,
n_iter=n_cases,
kappa=2)
best_solution = self.bayesian_optimizer.res['max']
self.best_solution_val = best_solution["max_val"]
self.best_solution_parameters = best_solution["max_params"].copy()
self.best_solution_parameters = self.parameter_bayesian_to_token(
self.best_solution_parameters)
self.best_solution_parameters = self.from_fit_params_to_saved_params[
frozenset(self.best_solution_parameters.items())]
writeLog(
"BayesianSearch: Best config is: Config {}, {} value is {:.4f}\n".
format(self.best_solution_parameters, metric,
self.best_solution_val), self.logFile)
#
#
# if folderPath != None:
#
# writeLog("BayesianSearch: Saving model in {}\n".format(folderPath), self.logFile)
# self.runSingleCase_param_parsed(dictionary, metric, self.best_solution_parameters, folderPath = folderPath, namePrefix = namePrefix)
return self.best_solution_parameters.copy()
#
# def evaluate_on_test(self):
#
# # Create an object of the same class of the imput
# # Passing the paramether as a dictionary
# recommender = self.recommender_class(*self.dictionary_input[DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS],
# **self.dictionary_input[DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS])
#
#
# recommender.fit(*self.dictionary_input[DictionaryKeys.FIT_POSITIONAL_ARGS],
# **self.dictionary_input[DictionaryKeys.FIT_KEYWORD_ARGS],
# **self.best_solution_parameters)
#
#
# result_dict = self.evaluator_test.evaluateRecommender(recommender, self.best_solution_parameters)
#
#
# writeLog("ParameterSearch: Best result evaluated on URM_test. Config: {} - results: {}\n".format(self.best_solution_parameters, result_dict), self.logFile)
#
# return result_dict
#
def parameter_bayesian_to_token(self, paramether_dictionary):
"""
The function takes the random values from BayesianSearch and transforms them in the corresponding categorical
tokens
:param paramether_dictionary:
:return:
"""
# Convert categorical values
for key in paramether_dictionary.keys():
if key in self.categorical_mapper_dict_index_to_case:
float_value = paramether_dictionary[key]
index = int(round(float_value, 0))
categorical = self.categorical_mapper_dict_index_to_case[key][
index]
paramether_dictionary[key] = categorical
return paramether_dictionary
def runSingleCase(self, dictionary, metric, **paramether_dictionary_input):
paramether_dictionary = self.parameter_bayesian_to_token(
paramether_dictionary_input)
return self.runSingleCase_param_parsed(dictionary, metric,
paramether_dictionary)
def runSingleCase_param_parsed(self, dictionary, metric,
paramether_dictionary):
try:
# Create an object of the same class of the imput
# Passing the paramether as a dictionary
recommender = self.recommender_class(
*dictionary[DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS],
**dictionary[DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS])
print("BayesianSearch: Testing config: {}".format(
paramether_dictionary))
recommender.fit(*dictionary[DictionaryKeys.FIT_POSITIONAL_ARGS],
**dictionary[DictionaryKeys.FIT_KEYWORD_ARGS],
**paramether_dictionary)
#return recommender.evaluateRecommendations(self.URM_validation, at=5, mode="sequential")
result_dict, _ = self.evaluator_validation.evaluateRecommender(
recommender, paramether_dictionary)
result_dict = result_dict[list(result_dict.keys())[0]]
paramether_dictionary_to_save = self.from_fit_params_to_saved_params_function(
recommender, paramether_dictionary)
self.from_fit_params_to_saved_params[frozenset(
paramether_dictionary.items())] = paramether_dictionary_to_save
self.model_counter += 1
# Always save best model separately
if self.save_model == "all":
print(self.ALGORITHM_NAME +
": Saving model in {}\n".format(self.output_root_path))
recommender.saveModel(self.output_root_path,
file_name="_model_{}".format(
self.model_counter))
pickle.dump(paramether_dictionary_to_save.copy(),
open(
self.output_root_path +
"_parameters_{}".format(self.model_counter),
"wb"),
protocol=pickle.HIGHEST_PROTOCOL)
if self.best_solution_val == None or self.best_solution_val < result_dict[
metric]:
writeLog(
"BayesianSearch: New best config found. Config: {} - results: {}\n"
.format(paramether_dictionary_to_save,
result_dict), self.logFile)
pickle.dump(paramether_dictionary_to_save.copy(),
open(self.output_root_path + "_best_parameters",
"wb"),
protocol=pickle.HIGHEST_PROTOCOL)
pickle.dump(
result_dict.copy(),
open(self.output_root_path + "_best_result_validation",
"wb"),
protocol=pickle.HIGHEST_PROTOCOL)
self.best_solution_val = result_dict[metric]
self.best_solution_parameters = paramether_dictionary_to_save.copy(
)
#self.best_solution_object = recommender
if self.save_model != "no":
print("BayesianSearch: Saving model in {}\n".format(
self.output_root_path))
recommender.saveModel(self.output_root_path,
file_name="_best_model")
if self.evaluator_test is not None:
self.evaluate_on_test()
else:
writeLog(
"BayesianSearch: Config is suboptimal. Config: {} - results: {}\n"
.format(paramether_dictionary_to_save,
result_dict), self.logFile)
return result_dict[metric]
except Exception as e:
writeLog(
"BayesianSearch: Testing config: {} - Exception {}\n".format(
paramether_dictionary, str(e)), self.logFile)
traceback.print_exc()
return -np.inf
traceback.print_exc()
return -np.inf
def function_interface(x, y):
return -x**2 - (y - 1)**2 + 1
if __name__ == '__main__':
# Lets find the maximum of a simple quadratic function of two variables
# We create the bayes_opt object and pass the function to be maximized
# together with the parameters names and their bounds.
bo = BayesianOptimization(function_interface, {'x': (-4, 4), 'y': (-3, 3)})
# One of the things we can do with this object is pass points
# which we want the algorithm to probe. A dictionary with the
# parameters names and a list of values to include in the search
# must be given.
bo.explore({'x': [-1, 3], 'y': [-2, 2]})
# Additionally, if we have any prior knowledge of the behaviour of
# the target function (even if not totally accurate) we can also
# tell that to the optimizer.
# Here we pass a dictionary with 'target' and parameter names as keys and a
# list of corresponding values
bo.initialize({'target': [-1, -1], 'x': [1, 1], 'y': [0, 2]})
# Once we are satisfied with the initialization conditions
bounds = [(1.0e-5, 1.0e-1), # learning rate
(0.5, 0.9999), # change of learning rate
(2, 1000)] # number of leaves
n_random_trials = 3 # initiate Bayesian optimization with 3 random draws
n_searches = 10
# Use my Bayesian Optimization
mdl = Model(data_mat, lags, n_oos, n_val, prediction_range,
target_vars_inds, params)
kernel = Kernel("rbf", 1)
bo = BayesianOptimization(mdl.obj_fun, bounds, kernel,
expected_improvement, n_random_trials)
ind, best_para_my, y = bo.search(n_searches, 2, 25)
# Use Ax Bayesian Optimization
n_random_trials = 5 # initiate Bayesian optimization with 3 random draws
n_searches = 20
mdl = Model(data_mat, lags, n_oos, n_val, prediction_range,
target_vars_inds, params)
search_space = SearchSpace(parameters=[
RangeParameter(name="lr", lower=1.0e-5, upper=1.0e-1,
def rfccv(n_estimators, min_samples_split):
return cross_val_score(RFC(n_estimators=int(n_estimators),
min_samples_split=int(min_samples_split),
random_state=2,
n_jobs=-1),
train, train_labels, 'roc_auc', cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = make_data(train_path = "../input/train.csv", test_path="../input/test.csv")
# RF
rfcBO = BayesianOptimization(rfccv, {'n_estimators': (600, 800),
'min_samples_split': (2, 5)})
print('-' * 53)
rfcBO.maximize()
print('-' * 53)
print('Final Results')
print('RFC: %f' % rfcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(RFC(n_estimators=int(rfcBO.res['max']['max_params']['n_estimators']),
min_samples_split=int(rfcBO.res['max']['max_params']['min_samples_split']),
random_state=2,
n_jobs=-1),
train, train_labels, 'roc_auc', cv=5).mean()
rf.fit(train, train_labels)
preds = rf.predict_proba(test)[:, 1]
random_state=2,
n_jobs=-1),
train,
train_labels,
'roc_auc',
cv=5).mean()
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = make_data(
train_path="../input/train.csv", test_path="../input/test.csv")
# RF
rfcBO = BayesianOptimization(rfccv, {
'n_estimators': (600, 800),
'min_samples_split': (2, 5)
})
print('-' * 53)
rfcBO.maximize()
print('-' * 53)
print('Final Results')
print('RFC: %f' % rfcBO.res['max']['max_val'])
# # MAKING SUBMISSION
rf = cross_val_score(RFC(
n_estimators=int(rfcBO.res['max']['max_params']['n_estimators']),
min_samples_split=int(
rfcBO.res['max']['max_params']['min_samples_split']),
random_state=2,
n_jobs=-1),
train,
import graph
from math import floor
from sa import sa
from qa import qa, qarev
import cycle
from bayesian_optimization import BayesianOptimization
def Bay_SA(t0, x):
Adj, E, edgno = graph.generate(100, 20)
sum = 0
for _ in range(3):
sum+= sa(100, E, 25, edgno, t0, 100, 1+int(floor(x)), 1+10-int(floor(x)))[0]
return -sum
#FOR BAYESIAN OPTIMIZATION
bo = BayesianOptimization(Bay_SA,{'t0': (0.00000000001, 1), 'x': (0.0000000001, 10)})
bo.maximize(init_points=15, n_iter=45, kappa=2)
print(bo.res['max'])
#floor(x) should converge to 0
from bayesian_optimization import UtilityFunction, BayesianOptimization
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from ThreeDEnv import environment_array
utility_function = UtilityFunction(kind="ei",xi=0,kappa=0)
def target(x,y,z):
return environment_array(x,y,z)
optimizer = BayesianOptimization(target, {'z': (2, 4),'y':(40,65),'x':(1000,4000)}, random_state=250)
#optimizer.maximize(init_points=int(input('Enter the number of random steps: ')),n_iter=0)
'''probe_point(np.min(x),np.max(y))
probe_point(np.max(x),np.max(y))
probe_point(np.max(x),np.min(y))'''
def probe_point(x,y,z):
return optimizer.probe(params={"x": x, "y": y, "z": z},lazy=True,)
probe_point(1000,40,2)
probe_point(1000,40,4)
probe_point(4000,40,4)
probe_point(4000,65,4)
probe_point(4000,65,2)
probe_point(1000,65,2)
probe_point(1000,65,4)
x = np.random.uniform(1000, 4000)
y = np.random.uniform(40, 65)
z = z
c = prediction(x, y, z)
i = i + 1
a.append(x)
b.append(y)
j.append(z)
cv.append(c)
i = i + 1
pbounds = {'y': (40, 65), 'z': (2, 4), 'x': (1000, 4000)}
optimizer = BayesianOptimization(
f=prediction,
pbounds=pbounds,
verbose=
1, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
random_state=999)
def probe_point(x, y, z):
return optimizer.probe(
params={
"x": x,
"y": y,
"z": z
},
lazy=True,
)
np.random.seed(0)
iterations_list = np.arange(1, 11)
for run_iter in iterations_list:
'''
The input arguments to "BayesianOptimization" are explained in the script "bayesian_optimization.py";
In particular, set "no_BOS=True" if we want to run standard GP-UCB, and "no_BOS=False" if we want to run the BO-BOS algorithm;
When running the "maximize" function, the intermediate results are saved after every BO iteration, under the file name log_file; the content of the log file is explained in the "analyze_results" ipython notebook script.
'''
# run without BOS
BO_no_BOS = BayesianOptimization(f=objective_function,
dim = 3, gp_opt_schedule=10, \
no_BOS=True, use_init=None, \
log_file="saved_results/bos_mnist_no_stop_" + str(run_iter) + ".p", save_init=True, \
save_init_file="mnist_5_" + str(run_iter) + ".p", \
parameter_names=["batch_size", "C", "learning_rate"])
# "parameter_names" are dummy variables whose correspondance in the display is not guaranteed
BO_no_BOS.maximize(n_iter=50,
init_points=3,
kappa=2,
use_fixed_kappa=False,
kappa_scale=0.2,
acq='ucb')
# run with BOS, using the same initializations as above
BO_BOS = BayesianOptimization(f=objective_function,
dim = 3, gp_opt_schedule=10, no_BOS=False, use_init="mnist_5_" + str(run_iter) + ".p", \
log_file="saved_results/bos_mnist_with_stop_" + str(run_iter) + ".p", save_init=False, \
save_init_file=None, \
b.append(y)
j.append(z)
cv.append(c)
i = i + 1
cv = np.array(cv)
'''Here is where the gaussian noise is created and added to the plot'''
'''noise = np.random.normal(0,100,cv.shape)
cv = cv +noise'''
img = ax.scatter(a, b, j, c=cv, cmap=plt.jet())
fig.colorbar(img)
print(max(cv))
pbounds = {'y': (40, 65), 'z': (2, 4), 'x': (1000, 4000)}
optimizer = BayesianOptimization(
f=black_box_function,
pbounds=pbounds,
verbose=
2, # verbose = 1 prints only when a maximum is observed, verbose = 0 is silent
random_state=249)
def probe_point(x, y, z):
return optimizer.probe(
params={
"x": x,
"y": y,
"z": z
},
lazy=True,
)
import numpy as np from bayesian_optimization import BayesianOptimization from functions import meno_rosenbrock dx = 2 bounds = np.matlib.repmat([[-5.], [10.]], 1, dx) opt_value = 0. n = 20 cv_iter = 10 n_iter = 20 y_bo = np.empty((1, cv_iter, n_iter)) err_bo = np.empty((1, cv_iter, n_iter)) times_bo = np.empty((1, cv_iter, n_iter)) for fold in range(cv_iter): bo = BayesianOptimization(lambda x: meno_rosenbrock(x), bounds, opt_value) y_best, tot_err, tot_time = bo.maximize(init_points=n, n_iter=n_iter, kappa=2) y_bo[0, fold] = y_best err_bo[0, fold] = tot_err times_bo[0, fold] = tot_time del bo, y_best, tot_err, tot_time
random_state=seed)
xgb_model = clf.fit(X_train, y_train, eval_metric="auc", eval_set=[(X_valid, y_valid)], early_stopping_rounds=20)
y_pred = xgb_model.predict_proba(X_valid)[:,1]
return auc(y_valid, y_pred)
if __name__ == "__main__":
# Load data set and target values
train, train_labels, test, test_labels = \
make_data(train_path = "../input/xtrain_v6.csv", test_path="../input/xtest_v6.csv")
xgboostBO = BayesianOptimization(xgboostcv,
{'max_depth': (8, 30),
'learning_rate': (0.8, 0.1),
'n_estimators': (250, 1500),
'gamma': (1., 0.01),
'min_child_weight': (2, 20),
'max_delta_step': (0., 0.3),
'subsample': (0.7, 0.85),
'colsample_bytree': (0.7, 0.85)
})
xgboostBO.maximize(init_points=7, restarts=50, n_iter=30)
print('-' * 53)
print('Final Results')
print('XGBOOST: %f' % xgboostBO.res['max']['max_val'])
# Build and Run on the full data set K-fold times for bagging
seeds = [1234, 5434, 87897, 123125, 88888]
