def Baye_search(self, func, space: list):
checkpoint_saver = CheckpointSaver(args.hyper_ckpt)
rlt = gp_minimize(
func,
dimensions=space,
n_calls=108,
n_random_starts=3,
# callback=[checkpoint_saver],
random_state=42)
logger.debug(rlt)
logger.debug(rlt)
plot_convergence(rlt)
return rlt
def Baye_search_resume(self, func, path: str, space):
assert os.path.exists(path)
ckpt = load(path)
checkpoint_saver = CheckpointSaver(args.hyper_ckpt)
rlt = gp_minimize(
func,
dimensions=space,
x0=ckpt.x_iters,
y0=ckpt.func_vals,
n_calls=20,
n_random_starts=3,
# callback=[checkpoint_saver],
random_state=42)
logger.debug(rlt)
plot_convergence(rlt)
return rlt
def test_checkpoint_saver():
checkpoint_path = "./test_checkpoint.pkl"
if os.path.isfile(checkpoint_path):
os.remove(checkpoint_path)
checkpoint_saver = CheckpointSaver(checkpoint_path, compress=9)
result = dummy_minimize(bench1, [(-1.0, 1.0)],
callback=checkpoint_saver,
n_calls=10)
assert os.path.exists(checkpoint_path)
assert load(checkpoint_path).x == result.x
if os.path.isfile(checkpoint_path):
os.remove(checkpoint_path)
def xgboost_grid(x, y):
try:
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=42)
except:
x = list(x)
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=42)
params = {
"objective": ['reg:squarederror'],
"colsample_bytree": [0.25, 0.5, 0.75],
"learning_rate": [0.01, 0.1, 0.2, 0.3],
"max_depth": [10, 20, 50],
"gamma": [i * 0.05 for i in range(0, 5)],
"lambda": [i * 0.05 for i in range(0, 4)],
"alpha": [i * 0.05 for i in range(0, 4)],
"eta": [i * 0.05 for i in range(0, 4)],
"n_estimators": [400, 4000],
"tree_method": ["gpu_hist"]
}
xgb_temp = xgb.XGBRegressor()
reg = GridSearchCV(xgb_temp, params, verbose=5, cv=3)
time_to_stop = 60 * 60
ckpt_loc = "../data/train/bayes/ckpt_bayes_xgboost.pkl"
checkpoint_callback = CheckpointSaver(ckpt_loc)
reg.fit(x_train,
y_train,
callback=[DeadlineStopper(time_to_stop), checkpoint_callback])
print(reg.best_params_)
print(reg.best_score_)
return reg
def optimize():
opt = TrainOptions().parse() # get training options
search_space = [
Real(0, 30, name='lambda_A'),
Real(0, 30, name='lambda_B'),
Real(0, 0.1, name='lambda_identity')
]
checkpoint_saver = CheckpointSaver("./checkpoint_optimization.pkl",
store_objective=False,
compress=9)
@use_named_args(search_space)
def objective_fn(lambda_A, lambda_B, lambda_identity):
return train_evaluate(lambda_A, lambda_B, lambda_identity, opt)
res = gp_minimize(
objective_fn,
search_space,
acq_func="EI", # the acquisition function
n_calls=15, # the number of evaluations of f
n_random_starts=5, # the number of random initialization points
callback=[checkpoint_saver])
return res
def optimize(self, num_iter, resume=False, n_initial_points=10, **kwargs):
'''Run BGO optimizer to find optimal parameters to fit against target diffuse'''
if resume:
cres = skopt.load(self.output_fname)
x_init = cres.x_iters
y_init = cres.func_vals
n_initial_points = 0
num_iter += len(cres.x_iters)
else:
x_init = None
y_init = None
checkpoint_saver = CheckpointSaver(self.output_fname,
store_objective=False)
skopt.gp_minimize(self.obj_fun,
self.dims,
n_calls=num_iter,
n_random_starts=n_initial_points,
callback=[checkpoint_saver],
noise=1e-7,
verbose=True,
x0=x_init,
y0=y_init,
**kwargs)
noise_level = 0.1
if IS_RUN_WITH_SPHINX_GALLERY:
# When this example is run with sphinx gallery, it breaks the pickling
# capacity for multiprocessing backend so we have to modify the way we
# define our functions. This has nothing to do with the example.
from utils import obj_fun
else:
def obj_fun(x, noise_level=noise_level):
return np.sin(5 * x[0]) * (
1 - np.tanh(x[0]**2)) + np.random.randn() * noise_level
checkpoint_saver = CheckpointSaver(
"./checkpoint.pkl",
compress=9) # keyword arguments will be passed to `skopt.dump`
gp_minimize(
obj_fun, # the function to minimize
[(-20.0, 20.0)], # the bounds on each dimension of x
x0=[-20.], # the starting point
acq_func="LCB", # the acquisition function (optional)
n_calls=10, # the number of evaluations of f including at x0
n_random_starts=0, # the number of random initialization points
callback=[checkpoint_saver
], # a list of callbacks including the checkpoint saver
random_state=777)
#############################################################################
# Now let's assume this did not finish at once but took some long time: you
clf = RandomForestClassifier(n_estimators=20, random_state=41)
# specify parameters and distributions to sample from
from skopt.space import Real, Categorical, Integer
param_dist = {
"max_depth": Categorical([3, None]),
"max_features": Integer(1, 11),
"min_samples_split": Integer(2, 11),
"min_samples_leaf": Integer(1, 11),
"bootstrap": Categorical([True, False]),
"criterion": Categorical(["gini", "entropy"])
}
# callback for saving checkopoints
from skopt.callbacks import CheckpointSaver
checkpoint_callback = CheckpointSaver("./result.pkl")
# callback for monitoring results and for early termination
def montitoring_callback(res):
current_step = len(searchcv.cv_results_['mean_test_score'])
current_score = searchcv.cv_results_['mean_test_score'][-1]
current_params = list(searchcv.cv_results_['params'][-1].values())
best_step = searchcv.best_index_ + 1
score = searchcv.best_score_
best_params = searchcv.best_params_
print("Step %s: Params: %s. Score: %s. Best step is %s" %
(current_step, current_params, current_score, best_step))
from skopt.space import Real, Integer
import skopt
from skopt import gp_minimize
from skopt.utils import use_named_args
from skopt import callbacks
from skopt.callbacks import CheckpointSaver
from skopt import load
checkpoint_saver = CheckpointSaver(
"C:\\Users\\denma\\Documents\\Uni\\Thesis\\Simulator\\optimising_TSHD_path\\Sim\\check\\checkpoint.pkl",
)
def main(from_beginning, verbose, visualization, check_embedding, remove,
num_epochs, start_epoch, patience, edge_batch_size, item_id_type,
duplicates):
"""
Main function that loads data and parameters, then runs hyperparameter loop with the fitness function.
"""
if verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
data_paths = DataPaths()
fixed_params = FixedParameters(num_epochs, start_epoch, patience,
edge_batch_size, remove, item_id_type,
duplicates)
checkpoint_saver = CheckpointSaver(
f'checkpoint{str(datetime.datetime.now())[:-10]}.pkl', compress=9)
data = DataLoader(data_paths, fixed_params)
global fitness_params
fitness_params = {
'data': data,
'fixed_params': fixed_params,
'data_paths': data_paths,
'visualization': visualization,
'check_embedding': check_embedding,
}
if from_beginning:
search_result = gp_minimize(
func=fitness,
dimensions=searchable_params.dimensions,
n_calls=200,
acq_func='EI',
x0=searchable_params.default_parameters,
callback=[checkpoint_saver],
random_state=46,
)
if not from_beginning:
checkpoint_path = None
if checkpoint_path is None:
checkpoint_path = get_last_checkpoint()
res = load(checkpoint_path)
x0 = res.x_iters
y0 = res.func_vals
search_result = gp_minimize(
func=fitness,
dimensions=searchable_params.dimensions,
n_calls=200,
n_initial_points=-len(
x0
), # Workaround suggested to correct the error when resuming training
acq_func='EI',
x0=x0,
y0=y0,
callback=[checkpoint_saver],
random_state=46)
log.info(search_result)
