def hyper_parameter_tuning_sge(n_step,n_init_sample,eval_type='dict', max_eval_each=100000, problem_set=['housing'],para_list='/util/hyper_para_list_SGE.json'):
root_dir=os.getcwd()
os.chdir("SGE/src/")
if not os.path.exists('../log'):
os.mkdir('../log')
for problem in problem_set:
minimize_problem=True
# by default, it will be a minimize problem.
paralist_filename = root_dir + para_list
system_name = 'SGE'
#main parameter
POPULATION_SIZE = get_space('POPULATION_SIZE', filename=paralist_filename, system_name=system_name)
ELITISM = get_space('ELITISM', filename=paralist_filename, system_name=system_name)
TOURNAMENT = get_space('TOURNAMENT', filename=paralist_filename, system_name=system_name)
PROB_CROSSOVER = get_space('PROB_CROSSOVER', filename=paralist_filename, system_name=system_name)
PROB_MUTATION = get_space('PROB_MUTATION', filename=paralist_filename, system_name=system_name)
# MAX_REC_LEVEL = get_space('MAX_REC_LEVEL', filename=paralist_filename, system_name=system_name)
#others/Static parameters
EVAL_BUDGET = OrdinalSpace([max_eval_each, max_eval_each + 1], 'EVAL_BUDGET')
PROBLEM = NominalSpace([problem], 'PROBLEM')
NUMBER_OF_ITERATIONS=OrdinalSpace([50, 50 + 1], 'NUMBER_OF_ITERATIONS')
search_space = PROBLEM + POPULATION_SIZE + ELITISM + TOURNAMENT + PROB_CROSSOVER + PROB_MUTATION + NUMBER_OF_ITERATIONS + EVAL_BUDGET
model = RandomForest(levels=search_space.levels)
opt = BO(search_space, obj_func, model, max_iter=n_step,
n_init_sample=n_init_sample,
n_point=1, # number of the candidate solution proposed in each iteration
n_job=1, # number of processes for the parallel execution
minimize=minimize_problem,
eval_type=eval_type, # use this parameter to control the type of evaluation
verbose=True, # turn this off, if you prefer no output
optimizer='MIES')
xopt, fitness, stop_dict = opt.run()
f = open(r"../../logs/out_SGE_" + problem + '_' + str(int(time.time())) + platform.uname()[1] + ".txt", 'w+')
print('parameters: {}'.format(search_space.name), file=f)
print('xopt: {}'.format(xopt), file=f)
print('fopt: {}'.format(fitness), file=f)
print('stop criteria: {}'.format(stop_dict), file=f)
f.close()
def __call__(self,
fid,
target,
budget,
dim=5,
seed=0,
log_file=None,
data_file=None,
verbose=True):
def obj_func(x):
print(x)
conf = reprToInt(x[3:])
return single_split_with_hyperparams_parallel(
fid,
dim,
conf,
conf,
51,
iids=[1, 2, 3, 4, 5],
num_reps=5,
hyperparams=[self.params, x[:3]],
hyperparams2=None,
budget=budget,
target_idx=target)
model = RandomForest()
opt = BO(self.search_space,
obj_func,
model,
max_iter=self.max_iter,
n_init_sample=self.n_init_sample,
minimize=True,
verbose=verbose,
wait_iter=10,
random_seed=seed,
n_point=self.n_point,
optimizer='MIES',
log_file=log_file,
data_file=data_file)
return opt.run()
def iterate_BO_cycle(search_space, problem, model, max_eval, verbose, log_file,
random_seed, data_file, warm_data):
opt = BO(search_space,
problem,
model,
minimize=True,
max_eval=max_eval,
infill='EI',
n_init_sample=N_INIT_SAMPLES,
n_point=1,
n_job=1,
optimizer='MIES',
eval_type='dict',
verbose=verbose,
log_file=log_file,
random_seed=random_seed,
data_file=data_file,
warm_data=warm_data)
best_sol_as_list, fitness, stop_dict = opt.run()
if verbose: print(stop_dict)
best_sol = opt.xopt.to_dict()
return best_sol, fitness, opt.data
def test_BO(dim, obj_fun, ftarget, max_FEs, lb, ub, logfile):
sys.path.insert(0, "../")
sys.path.insert(0, "../../GaussianProcess")
from BayesOpt import BO, DiscreteSpace, IntegerSpace, RandomForest, RealSpace
from GaussianProcess import GaussianProcess
from GaussianProcess.trend import constant_trend
space = RealSpace([lb, ub]) * dim
# kernel = 1.0 * Matern(length_scale=(1, 1), length_scale_bounds=(1e-10, 1e2))
# model = _GaussianProcessRegressor(kernel=kernel, alpha=0, n_restarts_optimizer=30, normalize_y=False)
mean = constant_trend(dim, beta=0) # equivalent to Simple Kriging
thetaL = 1e-5 * (ub - lb) * np.ones(dim)
thetaU = 10 * (ub - lb) * np.ones(dim)
theta0 = np.random.rand(dim) * (thetaU - thetaL) + thetaL
model = GaussianProcess(
mean=mean,
corr="matern",
theta0=theta0,
thetaL=thetaL,
thetaU=thetaU,
noise_estim=False,
nugget=0,
optimizer="BFGS",
wait_iter=5,
random_start=10 * dim,
eval_budget=200 * dim,
)
return BO(
search_space=space,
obj_fun=obj_fun,
model=model,
DoE_size=dim * 10,
max_FEs=max_FEs,
verbose=True,
n_point=1,
minimize=True,
acquisition_fun="EI",
ftarget=ftarget,
logger=None,
)
**etc_params)
print("Constraint: " + str(flags.constraint))
constraint_eq_func = mrs_problem.penalty if flags.constraint else None
#TODO pass mrs_problem.mrs_fit as the obj_func
#opt = mipego(
opt = BO(
search_space,
mrs_problem.mrs_fit,
model,
minimize=False,
eq_func=constraint_eq_func,
max_eval=opt_params['max_eval'],
max_iter=opt_params['max_iter'],
infill='EI', #Expected improvement as criteria
n_init_sample=opt_params[
'n_init_samples'], #We start with 10 initial samples
n_point=1, #We evaluate every iteration 1 time
n_job=1, # with 1 process (job).
optimizer='MIES', #We use the MIES internal optimizer.
eval_type='dict', #To get the solution, as well as the var_name
verbose=flags.verbose,
log_file=etc_params['log_filename'],
random_seed=flags.seed,
data_file=etc_params['data_filename'],
warm_data=flags.warmdata) # mipego -> warm_data_file=flags.warmdata)
print("### Begin Optimization ######################################")
incumbent_list, fitness, stop_dict = opt.run()
print(stop_dict)
x = opt.xopt.to_dict()
print("Best solution: " + str(x) + ", Fitness: " + str(fitness))
return np.sum(x**2)
space = ContinuousSpace([lb, ub]) * dim
mean = constant_trend(dim, beta=None)
thetaL = 1e-10 * (ub - lb) * np.ones(dim)
thetaU = 10 * (ub - lb) * np.ones(dim)
theta0 = np.random.rand(dim) * (thetaU - thetaL) + thetaL
model = GaussianProcess(corr='squared_exponential',
theta0=theta0,
thetaL=thetaL,
thetaU=thetaU,
nugget=0,
noise_estim=False,
optimizer='BFGS',
wait_iter=3,
random_start=dim,
likelihood='concentrated',
eval_budget=100 * dim)
opt = BO(search_space=space,
obj_fun=fitness,
model=model,
DoE_size=5,
max_FEs=50,
verbose=True,
n_point=1)
print(opt.run())
last_y = -1
penalty = 0
_id = ""
for ix, p in enumerate(zip(x_i, f_d)):
n, f = p
if f == 'Y':
penalty += ix - last_y - 1
last_y = last_y + 1
_id += str(n) if len(_id) == 0 else '-' + str(n)
penalty = (len(f_d) * (len(f_d) + 1)) / 2 if last_y == -1 else penalty
penalty = (len(f_d) * (len(f_d) + 1)) / 2 if _id in CACHE else penalty
return int(penalty)
space = (OrdinalSpace([1, 3]) * LENGTH) + (NominalSpace(['Y', 'N']) * LENGTH)
model = RandomForest(levels=space.levels)
opt = BO(space,
obj_func,
model,
eq_fun=eq_func,
ineq_fun=None,
minimize=True,
n_init_sample=3,
max_eval=50,
verbose=True,
optimizer='MIES')
xopt, fopt, stop_dict = opt.run()
print(xopt, fopt, stop_dict)
def hyper_parameter_tuning_GGES(n_step,
n_init_sample,
eval_type='dict',
max_eval_each=100000,
problem_set=['ant', 'string_match', 'mux11'],
para_list='/util/hyper_para_list_GGES.json'):
root_dir = os.getcwd()
os.chdir("GGES")
os.system("make ")
if not os.path.exists('log'):
os.mkdir('log')
for problem in problem_set:
minimize_problem = True
# by default, it will be a minimize problem.
paralist_filename = root_dir + para_list
system_name = 'GGES'
#main parameters
POPULATION_SIZE = get_space('pop_size',
filename=paralist_filename,
system_name=system_name)
TOURNAMENT_SIZE = get_space('tourn_size',
filename=paralist_filename,
system_name=system_name)
PROB_CROSSOVER = get_space('crossover_rate',
filename=paralist_filename,
system_name=system_name)
PROB_MUTATION = get_space('mutation_rate',
filename=paralist_filename,
system_name=system_name)
INIT_CODON_COUNT = get_space('init_codon_count',
filename=paralist_filename,
system_name=system_name)
REPRESENTATION = get_space('representation',
filename=paralist_filename,
system_name=system_name)
# SEARCH_METHOD = get_space('search_method',filename=paralist_filename, system_name=system_name)
#others/Static parameters
EVAL_BUDGET = OrdinalSpace([max_eval_each, max_eval_each + 1],
'EVAL_BUDGET')
PROBLEM = NominalSpace([problem], 'PROBLEM')
GENERATIONS = OrdinalSpace([50, 50 + 1], 'GENERATIONS')
search_space = PROBLEM + POPULATION_SIZE + TOURNAMENT_SIZE + PROB_CROSSOVER + PROB_MUTATION + INIT_CODON_COUNT + REPRESENTATION + GENERATIONS + EVAL_BUDGET
model = RandomForest(levels=search_space.levels)
opt = BO(
search_space,
obj_func,
model,
max_iter=n_step,
n_init_sample=n_init_sample,
n_point=
1, # number of the candidate solution proposed in each iteration
n_job=1, # number of processes for the parallel execution
minimize=minimize_problem,
eval_type=
eval_type, # use this parameter to control the type of evaluation
verbose=True, # turn this off, if you prefer no output
optimizer='MIES')
xopt, fitness, stop_dict = opt.run()
f = open(
r"../logs/out_GGES_" + problem + '_' + str(int(time.time())) +
platform.uname()[1] + ".txt", 'w+')
print('parameters: {}'.format(search_space.name), file=f)
print('xopt: {}'.format(xopt), file=f)
print('fopt: {}'.format(fitness), file=f)
print('stop criteria: {}'.format(stop_dict), file=f)
f.close()
def ineq_func(x):
x_r = np.array(x[:2])
return np.sum(x_r) + 1
space = (ContinuousSpace([-10, 10]) * 2) + OrdinalSpace([5, 15]) + \
NominalSpace(['OK', 'A', 'B', 'C', 'D', 'E', 'F', 'G'])
if 11 < 2:
opt = MIES(space, obj_func, eq_func=eq_func, ineq_func=ineq_func,
max_eval=1e3, verbose=True)
xopt, fopt, stop_dict = opt.optimize()
else:
model = RandomForest(levels=space.levels)
opt = BO(space, obj_func, model, eq_fun=None, ineq_fun=None,
minimize=True,
n_init_sample=3, max_eval=50, verbose=True, optimizer='MIES')
xopt, fopt, stop_dict = opt.run()
# if 11 < 2:
# N = int(50)
# max_eval = int(1000)
# fopt = np.zeros((1, N))
# hist_sigma = np.zeros((100, N))
# hist_fitness = np.zeros((100, N))
# for i in range(N):
# opt = mies(space, sphere, max_eval=max_eval, verbose=False)
# xopt, fopt[0, i], stop_dict, hist_fitness[:, i], hist_sigma[:, i] = opt.optimize()
# import matplotlib.pyplot as plt
def __call__(self,
fid,
dim,
rep1,
rep2,
split_idx,
iids=[1, 2, 3, 4, 5],
num_reps=5,
budget=None,
target_idx=None,
sol_points=None,
seed=0,
verbose=False,
log_file=None,
data_file=None,
opt_split=False):
np.random.seed(seed)
params = self.params
if self.part_to_optimize == 1 or self.part_to_optimize == -1:
initial_point = get_default_hyperparameter_values(
params, dim, rep1, budget)
else:
initial_point = get_default_hyperparameter_values(
params, dim, rep2, budget)
if sol_points is None:
initial_points = [initial_point]
else:
if isinstance(sol_points[0], list):
initial_points = sol_points.append(initial_point)
else:
if initial_point != sol_points:
initial_points = [sol_points, initial_point]
else:
initial_points = [initial_point]
if self.param_vals is not None and self.param_vals not in initial_points:
initial_points = initial_points.append(self.param_vals)
def obj_func(x):
if self.contains_discrete:
lambda1_ = x[-1]
lambda2_ = x[-1]
x = x[:-1]
params_i = [x for x in self.params if x != "lambda_"]
else:
lambda1_ = None
lambda2_ = None
params_i = self.params
if self.part_to_optimize == 1 or self.part_to_optimize == -1:
c1 = (params_i, x)
elif self.param_vals is not None:
if self.contains_discrete:
lambda1_ = self.param_vals[-1]
c1 = (params_i, self.param_vals[:-1])
else:
c1 = (params_i, self.param_vals)
else:
c1 = None
if self.part_to_optimize == 2 or self.part_to_optimize == -1:
c2 = (params_i, x)
elif self.param_vals is not None:
if self.contains_discrete:
lambda2_ = self.param_vals[-1]
c2 = (params_i, self.param_vals[:-1])
else:
c2 = (params_i, self.param_vals)
else:
c2 = None
print(c1, c2, lambda1_, lambda2_)
return single_split_with_hyperparams_parallel(
fid,
dim,
rep1,
rep2,
split_idx,
iids=iids,
num_reps=num_reps,
hyperparams=c1,
hyperparams2=c2,
budget=budget,
target_idx=target_idx,
lambda_=lambda1_,
lambda2_=lambda2_,
opt_split=opt_split)
if self.contains_discrete:
model = RandomForest()
opt = BO(self.search_space,
obj_func,
model,
max_iter=self.max_iter,
n_init_sample=self.n_init_sample,
minimize=True,
verbose=verbose,
wait_iter=10,
init_sol_points=initial_points,
random_seed=seed,
n_point=self.n_point,
optimizer='MIES',
log_file=log_file,
data_file=data_file)
else:
model = GaussianProcess(mean=self.mean,
corr='matern',
theta0=self.theta0,
thetaL=self.thetaL,
thetaU=self.thetaU,
nugget=1e-10,
noise_estim=False,
optimizer='BFGS',
wait_iter=5,
random_start=10 * self.dim_hyperparams,
likelihood='concentrated',
eval_budget=self.eval_budget,
random_state=seed)
opt = BO(
self.search_space,
obj_func,
model,
max_iter=self.max_iter,
n_init_sample=self.n_init_sample,
minimize=True,
verbose=verbose,
wait_iter=10,
init_sol_points=initial_points,
random_seed=seed,
n_point=self.n_point,
optimizer='BFGS',
log_file=log_file,
data_file=
data_file # when using GPR model, 'BFGS' is faster than 'MIES'
)
return opt.run()
def eq_func(x):
x_r = np.array(x[:2])
return np.sum(x_r ** 2) - 2
def ineq_func(x):
x_r = np.array(x[:2])
return np.sum(x_r) + 1
space = ((ContinuousSpace([-10, 10]) * 2) + OrdinalSpace([5, 15])
+ NominalSpace(['OK', 'A', 'B', 'C', 'D', 'E', 'F', 'G']))
warm_data = Solution([4.6827082694127835, 9.87885354178838, 5, 'A'], var_name=["r_0", "r_1", "i", "d"], n_eval=1, fitness=236.76575128)
warm_data += Solution([-8.99187067168115, 8.317469942991558, 5, 'D'], var_name=["r_0", "r_1", "i", "d"], n_eval=1, fitness=206.33644151)
warm_data += Solution([-2.50919762305275, 9.014286128198322, 12, 'G'], var_name=["r_0", "r_1", "i", "d"], n_eval=1, fitness=142.57378113)
warm_data += Solution([4.639878836228101, 1.973169683940732, 9, 'G'], var_name=["r_0", "r_1", "i", "d"], n_eval=1, fitness=70.8740683)
if 11 < 2:
model = RandomForest(levels=space.levels)
opt = BO(space, obj_func, model, minimize=True,
n_init_sample=3, max_eval=50, verbose=True, optimizer='MIES',
warm_data=warm_data)
xopt, fopt, stop_dict = opt.run()
else:
model = RandomForest(levels=space.levels)
opt = BO(space, obj_func, model, minimize=True,
n_init_sample=3, max_eval=50, verbose=True, optimizer='MIES',
warm_data="test_warmdata.data")
xopt, fopt, stop_dict = opt.run()
def hyper_parameter_tuning_ponyge2(
n_step,
n_init_sample,
eval_type,
max_eval_each=100000,
problem_set=['string_match'],
para_list='/util/hyper_para_list_PonyGE2.json'):
np.random.seed(67)
root_dir = os.getcwd()
os.chdir("PonyGE2/src/")
# if not os.path.exists('../log'):
# os.mkdir('../log')
for problem in problem_set:
# test problems one by one.
minimize_problem = True
# by default, it will be a minimize problem.
if problem in ['classification', 'pymax']:
minimize_problem = True
filename = root_dir + para_list
system_name = 'PonyGE2'
#Tunable hyper-parameters
INITIALISATION = get_space('INITIALISATION',
filename=filename,
system_name=system_name)
CROSSOVER = get_space('CROSSOVER',
filename=filename,
system_name=system_name)
CROSSOVER_PROBABILITY = get_space('CROSSOVER_PROBABILITY',
filename=filename,
system_name=system_name)
MUTATION = get_space('MUTATION',
filename=filename,
system_name=system_name)
MUTATION_PROBABILITY = get_space('MUTATION_PROBABILITY',
filename=filename,
system_name=system_name)
MUTATION_EVENT_SUBTREE = get_space('MUTATION_EVENT_SUBTREE',
filename=filename,
system_name=system_name)
MUTATION_EVENT_FlIP = get_space('MUTATION_EVENT_FlIP',
filename=filename,
system_name=system_name)
SELECTION_PROPORTION = get_space('SELECTION_PROPORTION',
filename=filename,
system_name=system_name)
SELECTION = get_space('SELECTION',
filename=filename,
system_name=system_name)
TOURNAMENT_SIZE = get_space('TOURNAMENT_SIZE',
filename=filename,
system_name=system_name)
ELITE_SIZE = get_space('ELITE_SIZE',
filename=filename,
system_name=system_name)
CODON_SIZE = get_space('CODON_SIZE',
filename=filename,
system_name=system_name)
MAX_GENOME_LENGTH = get_space('MAX_GENOME_LENGTH',
filename=filename,
system_name=system_name)
MAX_INIT_TREE_DEPTH = get_space('MAX_INIT_TREE_DEPTH',
filename=filename,
system_name=system_name)
MAX_TREE_DEPTH = get_space('MAX_TREE_DEPTH',
filename=filename,
system_name=system_name)
POPULATION_SIZE = get_space('POPULATION_SIZE',
filename=filename,
system_name=system_name)
# Others/Static parameters
PROBLEM = NominalSpace([problem], 'PROBLEM')
EVAL_BUDGET = OrdinalSpace([max_eval_each, max_eval_each + 1],
'EVAL_BUDGET')
#todo: By default, the following line should be 'if minimize_problem == True:', since the 'MAX_INIT_TREE_DEPTH' and 'MAX_TREE_DEPTH' can easily causes problem by generating too big search space.
#But we found it also prodeces this problem for mux11 problem. so these two parameters are temporarily disbaled.
if minimize_problem == False:
search_space = PROBLEM + INITIALISATION + CROSSOVER_PROBABILITY + CROSSOVER + MUTATION + MUTATION_PROBABILITY + MUTATION_EVENT_SUBTREE + MUTATION_EVENT_FlIP + SELECTION_PROPORTION + SELECTION + TOURNAMENT_SIZE + ELITE_SIZE + CODON_SIZE + MAX_GENOME_LENGTH + MAX_INIT_TREE_DEPTH + MAX_TREE_DEPTH + POPULATION_SIZE + EVAL_BUDGET
else:
# for maximize problem, Max_init_tree_depth and Max_tree_depth is not going to be tuned.
search_space = PROBLEM + INITIALISATION + CROSSOVER_PROBABILITY + CROSSOVER + MUTATION + MUTATION_PROBABILITY + MUTATION_EVENT_SUBTREE + MUTATION_EVENT_FlIP + SELECTION_PROPORTION + SELECTION + TOURNAMENT_SIZE + ELITE_SIZE + CODON_SIZE + MAX_GENOME_LENGTH + POPULATION_SIZE + EVAL_BUDGET
model = RandomForest(levels=search_space.levels)
opt = BO(
search_space,
obj_func,
model,
max_iter=n_step,
n_init_sample=n_init_sample,
n_point=
1, # number of the candidate solution proposed in each iteration
n_job=1, # number of processes for the parallel execution
minimize=minimize_problem,
eval_type=
eval_type, # use this parameter to control the type of evaluation
verbose=True, # turn this off, if you prefer no output
optimizer='MIES')
xopt, fitness, stop_dict = opt.run()
f = open(
r"../../logs/out_PonyGE2_" + problem + '_' +
str(int(time.time())) + platform.uname()[1] + ".txt", 'w+')
print('parameters: {}'.format(search_space.name), file=f)
print('xopt: {}'.format(xopt), file=f)
print('fopt: {}'.format(fitness), file=f)
print('stop criteria: {}'.format(stop_dict), file=f)
f.close()