def json2space(para_dict):
if para_dict['type'] == 'NominalSpace':
locals()[para_dict['name']]=NominalSpace(para_dict['options'], para_dict['name'])
if para_dict['type'] == 'ContinuousSpace':
locals()[para_dict['name']]=ContinuousSpace(para_dict['range'], para_dict['name'])
if para_dict['type'] == 'OrdinalSpace':
locals()[para_dict['name']]=OrdinalSpace(para_dict['range'], para_dict['name'])
def hyper_parameter_tuning_sge(n_step,n_init_sample,eval_type='dict', max_eval_each=100000, problem_set=['housing'],para_list='/util/hyper_para_list_SGE.json'):
root_dir=os.getcwd()
os.chdir("SGE/src/")
if not os.path.exists('../log'):
os.mkdir('../log')
for problem in problem_set:
minimize_problem=True
# by default, it will be a minimize problem.
paralist_filename = root_dir + para_list
system_name = 'SGE'
#main parameter
POPULATION_SIZE = get_space('POPULATION_SIZE', filename=paralist_filename, system_name=system_name)
ELITISM = get_space('ELITISM', filename=paralist_filename, system_name=system_name)
TOURNAMENT = get_space('TOURNAMENT', filename=paralist_filename, system_name=system_name)
PROB_CROSSOVER = get_space('PROB_CROSSOVER', filename=paralist_filename, system_name=system_name)
PROB_MUTATION = get_space('PROB_MUTATION', filename=paralist_filename, system_name=system_name)
# MAX_REC_LEVEL = get_space('MAX_REC_LEVEL', filename=paralist_filename, system_name=system_name)
#others/Static parameters
EVAL_BUDGET = OrdinalSpace([max_eval_each, max_eval_each + 1], 'EVAL_BUDGET')
PROBLEM = NominalSpace([problem], 'PROBLEM')
NUMBER_OF_ITERATIONS=OrdinalSpace([50, 50 + 1], 'NUMBER_OF_ITERATIONS')
search_space = PROBLEM + POPULATION_SIZE + ELITISM + TOURNAMENT + PROB_CROSSOVER + PROB_MUTATION + NUMBER_OF_ITERATIONS + EVAL_BUDGET
model = RandomForest(levels=search_space.levels)
opt = BO(search_space, obj_func, model, max_iter=n_step,
n_init_sample=n_init_sample,
n_point=1, # number of the candidate solution proposed in each iteration
n_job=1, # number of processes for the parallel execution
minimize=minimize_problem,
eval_type=eval_type, # use this parameter to control the type of evaluation
verbose=True, # turn this off, if you prefer no output
optimizer='MIES')
xopt, fitness, stop_dict = opt.run()
f = open(r"../../logs/out_SGE_" + problem + '_' + str(int(time.time())) + platform.uname()[1] + ".txt", 'w+')
print('parameters: {}'.format(search_space.name), file=f)
print('xopt: {}'.format(xopt), file=f)
print('fopt: {}'.format(fitness), file=f)
print('stop criteria: {}'.format(stop_dict), file=f)
f.close()
def create_optimizer(dim, fitness, n_step, n_init_sample, model_type):
C = ContinuousSpace([-5, 5]) * 2
I = OrdinalSpace([-100, 100])
N = NominalSpace(['OK', 'A', 'B', 'C', 'D', 'E'])
search_space = C * I * N
levels = search_space.levels
if model_type == 'GP':
# thetaL = 1e-3 * (ub - lb) * np.ones(dim)
# thetaU = 10 * (ub - lb) * np.ones(dim)
# theta0 = np.random.rand(dim) * (thetaU - thetaL) + thetaL
#
# model = GaussianProcess(regr='constant', corr='matern',
# theta0=theta0, thetaL=thetaL,
# thetaU=thetaU, nugget=1e-5,
# nugget_estim=False, normalize=False,
# verbose=False, random_start = 15*dim,
# random_state=None)
pass
elif model_type == 'sklearn-RF':
min_samples_leaf = max(1, int(n_init_sample / 20.))
max_features = int(np.ceil(dim * 5 / 6.))
model = RandomForest(levels=levels,
n_estimators=100,
max_features=max_features,
min_samples_leaf=min_samples_leaf)
elif model_type == 'R-RF':
min_samples_leaf = max(1, int(n_init_sample / 20.))
max_features = int(np.ceil(dim * 5 / 6.))
model = RrandomForest(levels=levels,
n_estimators=100,
max_features=max_features,
min_samples_leaf=min_samples_leaf)
opt = BayesOpt(search_space,
fitness,
model,
max_iter=n_step,
random_seed=None,
n_init_sample=n_init_sample,
minimize=True,
optimizer='MIES')
return opt
def get_space(spacename,filename,system_name=None):
if system_name:
with open(filename, 'r') as load_f:
data = json.load(load_f)
for parameters in data:
if parameters['name']==spacename:
if parameters['type'] == 'NominalSpace':
return NominalSpace(parameters['options'],parameters['name'])
if parameters['type'] == 'ContinuousSpace':
return ContinuousSpace(parameters['range'],parameters['name'])
if parameters['type'] == 'OrdinalSpace':
return OrdinalSpace(parameters['range'],parameters['name'])
else:
print("No system name was givem")
return
#data_loader = opt_params['data_loader_class']()
data_loader.load(**data_loader_params)
dataset = data_loader.dataset
#Define the search space
search_space = None
model = None
print("Encoding: " + flags.encoding)
print("Repair: " + str(flags.repair))
if flags.encoding == 'flag':
cells_per_layer = OrdinalSpace(
[opt_params['min_nn'], opt_params['max_nn']],
'cells_per_layer') * opt_params['max_hl']
look_back = OrdinalSpace([opt_params['min_lb'], opt_params['max_lb']],
'look_back')
layer = NominalSpace(['Y', 'N'], 'layer') * opt_params['max_hl']
search_space = ProductSpace(ProductSpace(cells_per_layer, layer),
look_back)
# mipego -> search_space = cells_per_layer * layer * look_back
#assign the right decode function
solution_decoder = decode_solution_flag
model = RandomForest(levels=search_space.levels)
elif flags.encoding == 'size':
cells_per_layer = OrdinalSpace(
[opt_params['min_nn'], opt_params['max_nn']],
'cells_per_layer') * opt_params['max_hl']
look_back = OrdinalSpace([opt_params['min_lb'], opt_params['max_lb']],
'look_back')
size = OrdinalSpace([1, opt_params['max_hl']], 'size')
# size = NominalSpace(list(range(1, opt_params['max_hl']+1)), 'size')
search_space = ProductSpace(ProductSpace(cells_per_layer, size),
last_y = -1
penalty = 0
_id = ""
for ix, p in enumerate(zip(x_i, f_d)):
n, f = p
if f == 'Y':
penalty += ix - last_y - 1
last_y = last_y + 1
_id += str(n) if len(_id) == 0 else '-' + str(n)
penalty = (len(f_d) * (len(f_d) + 1)) / 2 if last_y == -1 else penalty
penalty = (len(f_d) * (len(f_d) + 1)) / 2 if _id in CACHE else penalty
return int(penalty)
space = (OrdinalSpace([1, 3]) * LENGTH) + (NominalSpace(['Y', 'N']) * LENGTH)
model = RandomForest(levels=space.levels)
opt = BO(space,
obj_func,
model,
eq_fun=eq_func,
ineq_fun=None,
minimize=True,
n_init_sample=3,
max_eval=50,
verbose=True,
optimizer='MIES')
xopt, fopt, stop_dict = opt.run()
print(xopt, fopt, stop_dict)
def hyper_parameter_tuning_GGES(n_step,
n_init_sample,
eval_type='dict',
max_eval_each=100000,
problem_set=['ant', 'string_match', 'mux11'],
para_list='/util/hyper_para_list_GGES.json'):
root_dir = os.getcwd()
os.chdir("GGES")
os.system("make ")
if not os.path.exists('log'):
os.mkdir('log')
for problem in problem_set:
minimize_problem = True
# by default, it will be a minimize problem.
paralist_filename = root_dir + para_list
system_name = 'GGES'
#main parameters
POPULATION_SIZE = get_space('pop_size',
filename=paralist_filename,
system_name=system_name)
TOURNAMENT_SIZE = get_space('tourn_size',
filename=paralist_filename,
system_name=system_name)
PROB_CROSSOVER = get_space('crossover_rate',
filename=paralist_filename,
system_name=system_name)
PROB_MUTATION = get_space('mutation_rate',
filename=paralist_filename,
system_name=system_name)
INIT_CODON_COUNT = get_space('init_codon_count',
filename=paralist_filename,
system_name=system_name)
REPRESENTATION = get_space('representation',
filename=paralist_filename,
system_name=system_name)
# SEARCH_METHOD = get_space('search_method',filename=paralist_filename, system_name=system_name)
#others/Static parameters
EVAL_BUDGET = OrdinalSpace([max_eval_each, max_eval_each + 1],
'EVAL_BUDGET')
PROBLEM = NominalSpace([problem], 'PROBLEM')
GENERATIONS = OrdinalSpace([50, 50 + 1], 'GENERATIONS')
search_space = PROBLEM + POPULATION_SIZE + TOURNAMENT_SIZE + PROB_CROSSOVER + PROB_MUTATION + INIT_CODON_COUNT + REPRESENTATION + GENERATIONS + EVAL_BUDGET
model = RandomForest(levels=search_space.levels)
opt = BO(
search_space,
obj_func,
model,
max_iter=n_step,
n_init_sample=n_init_sample,
n_point=
1, # number of the candidate solution proposed in each iteration
n_job=1, # number of processes for the parallel execution
minimize=minimize_problem,
eval_type=
eval_type, # use this parameter to control the type of evaluation
verbose=True, # turn this off, if you prefer no output
optimizer='MIES')
xopt, fitness, stop_dict = opt.run()
f = open(
r"../logs/out_GGES_" + problem + '_' + str(int(time.time())) +
platform.uname()[1] + ".txt", 'w+')
print('parameters: {}'.format(search_space.name), file=f)
print('xopt: {}'.format(xopt), file=f)
print('fopt: {}'.format(fitness), file=f)
print('stop criteria: {}'.format(stop_dict), file=f)
f.close()
if x_d == 'OK':
tmp = 0
else:
tmp = 1
return np.sum((x_r + np.array([2, 2])) ** 2) + abs(x_i - 10) * 10 + tmp
def eq_func(x):
x_r = np.array(x[:2])
return np.sum(x_r ** 2) - 2
def ineq_func(x):
x_r = np.array(x[:2])
return np.sum(x_r) + 1
space = (ContinuousSpace([-10, 10]) * 2) + OrdinalSpace([5, 15]) + \
NominalSpace(['OK', 'A', 'B', 'C', 'D', 'E', 'F', 'G'])
if 11 < 2:
opt = MIES(space, obj_func, eq_func=eq_func, ineq_func=ineq_func,
max_eval=1e3, verbose=True)
xopt, fopt, stop_dict = opt.optimize()
else:
model = RandomForest(levels=space.levels)
opt = BO(space, obj_func, model, eq_fun=None, ineq_fun=None,
minimize=True,
n_init_sample=3, max_eval=50, verbose=True, optimizer='MIES')
xopt, fopt, stop_dict = opt.run()
# if 11 < 2:
# N = int(50)
def __init__(self,
max_iter=None,
n_init_sample=None,
eval_budget=None,
n_random_start=None,
n_point=None):
"""
Creates a hyperparameter optimizer
:param fid: The function id (from bbob)
:param dim: The dimension to run the bbob-problem in
:param rep1: The configuration to run before the splitpoint
:param rep2: The configuration to run after the splitpoint
:param split_idx: The splitpoint-index at which to switch between rep1 and rep2
:param iids: The instances of the bbob-function to run
:param num_reps: The amount of repetitions to run
:param part_to_optimize: Which part of the adaptive configuration to optimize. Can be 1, 2 or -1. The -1
option optimizes both parts, with the same parameter values for both. To better optimize a complete
configuration, first optimize part 1, then part 2 using the optimial value for part1 in the param_val
argument.
:param param_val: The parameter values for the part of the configuration which is not optimized here.
:return: The result of the hyper-parameter optimzation
"""
self.params = [
'c_1', 'c_c', 'c_mu'
] #, 'm1', 'm2', 'm3', 'm4', 'm5', 'm6', 'm7', 'm8', 'm9', 'm10', 'm11']
self.dim_hyperparams = len(self.params)
if max_iter is None:
self.max_iter = 1000
else:
self.max_iter = max_iter
if n_init_sample is None:
self.n_init_sample = 250
else:
self.n_init_sample = n_init_sample
if eval_budget is None:
self.eval_budget = 20
else:
self.eval_budget = eval_budget
if n_random_start is None:
self.n_random_start = 5
else:
self.n_random_start = n_random_start
if n_point is None:
self.n_point = 1
else:
self.n_point = n_point
self.lb = np.zeros((3, 1))
self.ub = [0.35, 1, 0.35]
p1 = [[0, 1]] * 9
p1.append([0, 1, 2])
p1.append([0, 1, 2])
search_space_nominal = NominalSpace(p1)
# search_space_discrete = OrdinalSpace(list(zip([0,0,0,0,0,0,0,0,0,0,0], [2,2,2,2,2,2,2,2,2,3,3])))
search_space_cont = ContinuousSpace(
list(zip([0, 0, 0], [0.35, 1, 0.35])))
self.search_space = search_space_cont + search_space_nominal
self.mean = constant_trend(self.dim_hyperparams, beta=0)
# autocorrelation parameters of GPR
self.thetaL = 1e-10 * (self.ub - self.lb) * np.ones(
self.dim_hyperparams)
self.thetaU = 2 * (self.ub - self.lb) * np.ones(self.dim_hyperparams)
np.random.seed(0)
self.theta0 = np.random.rand(
self.dim_hyperparams) * (self.thetaU - self.thetaL) + self.thetaL
def hyper_parameter_tuning_ponyge2(
n_step,
n_init_sample,
eval_type,
max_eval_each=100000,
problem_set=['string_match'],
para_list='/util/hyper_para_list_PonyGE2.json'):
np.random.seed(67)
root_dir = os.getcwd()
os.chdir("PonyGE2/src/")
# if not os.path.exists('../log'):
# os.mkdir('../log')
for problem in problem_set:
# test problems one by one.
minimize_problem = True
# by default, it will be a minimize problem.
if problem in ['classification', 'pymax']:
minimize_problem = True
filename = root_dir + para_list
system_name = 'PonyGE2'
#Tunable hyper-parameters
INITIALISATION = get_space('INITIALISATION',
filename=filename,
system_name=system_name)
CROSSOVER = get_space('CROSSOVER',
filename=filename,
system_name=system_name)
CROSSOVER_PROBABILITY = get_space('CROSSOVER_PROBABILITY',
filename=filename,
system_name=system_name)
MUTATION = get_space('MUTATION',
filename=filename,
system_name=system_name)
MUTATION_PROBABILITY = get_space('MUTATION_PROBABILITY',
filename=filename,
system_name=system_name)
MUTATION_EVENT_SUBTREE = get_space('MUTATION_EVENT_SUBTREE',
filename=filename,
system_name=system_name)
MUTATION_EVENT_FlIP = get_space('MUTATION_EVENT_FlIP',
filename=filename,
system_name=system_name)
SELECTION_PROPORTION = get_space('SELECTION_PROPORTION',
filename=filename,
system_name=system_name)
SELECTION = get_space('SELECTION',
filename=filename,
system_name=system_name)
TOURNAMENT_SIZE = get_space('TOURNAMENT_SIZE',
filename=filename,
system_name=system_name)
ELITE_SIZE = get_space('ELITE_SIZE',
filename=filename,
system_name=system_name)
CODON_SIZE = get_space('CODON_SIZE',
filename=filename,
system_name=system_name)
MAX_GENOME_LENGTH = get_space('MAX_GENOME_LENGTH',
filename=filename,
system_name=system_name)
MAX_INIT_TREE_DEPTH = get_space('MAX_INIT_TREE_DEPTH',
filename=filename,
system_name=system_name)
MAX_TREE_DEPTH = get_space('MAX_TREE_DEPTH',
filename=filename,
system_name=system_name)
POPULATION_SIZE = get_space('POPULATION_SIZE',
filename=filename,
system_name=system_name)
# Others/Static parameters
PROBLEM = NominalSpace([problem], 'PROBLEM')
EVAL_BUDGET = OrdinalSpace([max_eval_each, max_eval_each + 1],
'EVAL_BUDGET')
#todo: By default, the following line should be 'if minimize_problem == True:', since the 'MAX_INIT_TREE_DEPTH' and 'MAX_TREE_DEPTH' can easily causes problem by generating too big search space.
#But we found it also prodeces this problem for mux11 problem. so these two parameters are temporarily disbaled.
if minimize_problem == False:
search_space = PROBLEM + INITIALISATION + CROSSOVER_PROBABILITY + CROSSOVER + MUTATION + MUTATION_PROBABILITY + MUTATION_EVENT_SUBTREE + MUTATION_EVENT_FlIP + SELECTION_PROPORTION + SELECTION + TOURNAMENT_SIZE + ELITE_SIZE + CODON_SIZE + MAX_GENOME_LENGTH + MAX_INIT_TREE_DEPTH + MAX_TREE_DEPTH + POPULATION_SIZE + EVAL_BUDGET
else:
# for maximize problem, Max_init_tree_depth and Max_tree_depth is not going to be tuned.
search_space = PROBLEM + INITIALISATION + CROSSOVER_PROBABILITY + CROSSOVER + MUTATION + MUTATION_PROBABILITY + MUTATION_EVENT_SUBTREE + MUTATION_EVENT_FlIP + SELECTION_PROPORTION + SELECTION + TOURNAMENT_SIZE + ELITE_SIZE + CODON_SIZE + MAX_GENOME_LENGTH + POPULATION_SIZE + EVAL_BUDGET
model = RandomForest(levels=search_space.levels)
opt = BO(
search_space,
obj_func,
model,
max_iter=n_step,
n_init_sample=n_init_sample,
n_point=
1, # number of the candidate solution proposed in each iteration
n_job=1, # number of processes for the parallel execution
minimize=minimize_problem,
eval_type=
eval_type, # use this parameter to control the type of evaluation
verbose=True, # turn this off, if you prefer no output
optimizer='MIES')
xopt, fitness, stop_dict = opt.run()
f = open(
r"../../logs/out_PonyGE2_" + problem + '_' +
str(int(time.time())) + platform.uname()[1] + ".txt", 'w+')
print('parameters: {}'.format(search_space.name), file=f)
print('xopt: {}'.format(xopt), file=f)
print('fopt: {}'.format(fitness), file=f)
print('stop criteria: {}'.format(stop_dict), file=f)
f.close()
n_step = 20
n_init_sample = 15
def obj_func(x):
x_r, x_i, x_d = np.array(x[:2]), x[2], x[3]
if x_d == 'OK':
tmp = 0
else:
tmp = 1
return np.sum(x_r**2.) + abs(x_i - 10) / 123. + tmp * 2.
C = ContinuousSpace([-5, 5]) * 2
I = OrdinalSpace([-100, 100])
N = NominalSpace(['OK', 'A', 'B', 'C', 'D', 'E'])
search_space = C * I * N
# thetaL = 1e-3 * (ub - lb) * np.ones(dim)
# thetaU = 10 * (ub - lb) * np.ones(dim)
# theta0 = np.random.rand(dim) * (thetaU - thetaL) + thetaL
# model = GaussianProcess(regr='constant', corr='matern',
# theta0=theta0, thetaL=thetaL,
# thetaU=thetaU, nugget=None,
# nugget_estim=False, normalize=False,
# verbose=False, random_start=15 * dim,
# random_state=None, optimizer='BFGS')
# min_samples_leaf = max(1, int(n_init_sample / 20.))