def tune_with_rbfopt(obj_name):
global simple_log
simple_log = []
settings = rbfopt.RbfoptSettings(
minlp_solver_path='\\Programs\\AMPL\\bonmin-win64\\bonmin.exe',
nlp_solver_path='\\Programs\\AMPL\\ipopt-win64\\ipopt.exe',
max_evaluations=100)
if obj_name == 'rosen':
param_dim = 8
obj_fc = rosen_eval
bb = rbfopt.RbfoptUserBlackBox(param_dim, np.array([-5] * param_dim),
np.array([5] * param_dim),
np.array(['R'] * param_dim), obj_fc)
elif obj_name == 'iris':
param_dim = 2
obj_fc = iris_eval
bb = rbfopt.RbfoptUserBlackBox(param_dim, np.array([-5] * param_dim),
np.array([5] * param_dim),
np.array(['R'] * param_dim), obj_fc)
elif obj_name == 'gbm':
bb = rbfopt.RbfoptUserBlackBox(dimension=2,
var_lower=np.array([1, -8]),
var_upper=np.array([3, 0]),
var_type=np.array(['R', 'R']),
obj_funct=gbm_eval)
else:
raise NotImplementedError
alg = rbfopt.RbfoptAlgorithm(settings, bb)
val, x, itercount, evalcount, fast_evalcount = alg.optimize()
history = simple_log
plot_simple_log(history)
return history
def main(args):
# example: bb = rbfopt.RbfoptUserBlackBox(3, np.array([0] * 3), np.array([10] * 3), np.array(['R', 'I', 'R']), obj_funct)
# data (hacky, no time to define properly right now)
global train_dir, model_dir, logging_fname, max_seq_len
if args.mode == "single":
train_dir = "training_data_single"
model_dir = "hockey_model_single"
logging_fname = "optimizer.logging.single"
max_seq_len = 50
elif args.mode == "combined_events":
train_dir = "training_data_combined_events"
model_dir = "hockey_model_combined_events"
logging_fname = "optimizer.logging.combined_events"
max_seq_len = 50
elif args.mode == "full_report":
train_dir = "training_data_full_report"
model_dir = "hockey_model_full_report"
logging_fname = "optimizer.logging.full_report"
max_seq_len = 300
else:
print("Unknown mode:", mode, file=sys.stderr)
sys.exit()
if os.path.exists(logging_fname):
os.remove(logging_fname)
files = glob.glob(model_dir + "/*")
for _f in files:
os.remove(_f)
# list of parameters:
# word embeddings (50-700) --word_vec_size
# recurrent size (300-1000) --rnn_size
# dropout (0.0-0.6) --dropout
# learning_rate (0.0001-0.01) --learning_rate
# enc/dec layers (1-3) --layers
# copy attention layer (1/0) --copy_attn --reuse_copy_attn
# coverage attention layer (1/0)--coverage_attn
# batch size (16/64) --batch_size
# --train_steps
# emb rnn drop lr layers copy cover batch
bb=rbfopt.RbfoptUserBlackBox(8,np.array([50, 300, 0.0, 0.00001, 1, 0, 0, 16]),\
np.array([700, 1000, 0.6, 0.01, 3, 1, 1, 64]),np.array(['I','I','R','R','I','I','I','I']), my_black_box)
settings = rbfopt.RbfoptSettings(
max_clock_time=36 * 60 * 60 * 1,
target_objval=0.0,
num_cpus=1,
minlp_solver_path='/home/jmnybl/optimizer_tools/bonmin',
nlp_solver_path=
'/home/jmnybl/optimizer_tools/Ipopt-3.7.1-linux-x86_64-gcc4.3.2/bin/ipopt'
)
alg = rbfopt.RbfoptAlgorithm(settings, bb)
val, x, itercount, evalcount, fast_evalcount = alg.optimize()
def run_rbf_optimization(
mus,
sigmas,
ids,
num_clusters=2,
num_objectives=3,
verbose=False,
optimization_iterations=10,
):
try:
import rbfopt
settings = rbfopt.RbfoptSettings(
minlp_solver_path='/home/amarildo/Bonmin-stable/build/bin/bonmin',
nlp_solver_path='/home/amarildo/Bonmin-stable/build/bin/ipopt')
num_agents = len(mus)
weight_calculator = make_weights_assignment_function(
mus,
sigmas,
ids,
num_objectives=num_objectives,
verbose=verbose,
num_iters=optimization_iterations)
# The objective function.
def obj_func(assignment):
assignment = assignment.reshape((num_clusters, num_agents))
obj = 0
for i in range(assignment.shape[0]):
w, loss = weight_calculator(assignment[i])
obj += loss
return obj
num_opt_params = num_clusters * num_agents
bb = rbfopt.RbfoptUserBlackBox(num_opt_params,
np.array([0] * num_opt_params),
np.array([1] * num_opt_params),
np.array(['R'] * num_opt_params),
obj_func)
alg = rbfopt.RbfoptAlgorithm(settings, bb)
val, x, itercount, evalcount, fast_evalcount = alg.optimize()
print("Value:", val)
print("Assignment:")
print(x)
print("Iteration Count:", itercount)
print("Evaluation Count:", evalcount)
print("Fast Evaluation Count:", fast_evalcount)
loss_value = 0.
best_weights = np.zeros(shape=(num_clusters, num_objectives))
best_assignment = x.reshape((num_clusters, num_agents))
for i in range(num_clusters):
w, loss = weight_calculator(best_assignment[i])
best_weights[i] = w
loss_value += loss
return best_assignment, best_weights, loss_value
except ImportError:
print("RBF optimizer not installed")
return 0, 0, 0
def test_minimal_working_example(self):
"""Check solution of minimal working example.
Construct the minimal working example described in the README,
and verify that it is correctly solved.
"""
bb = rbfopt.RbfoptUserBlackBox(3, np.array([0] * 3),
np.array([10] * 3),
np.array(['R', 'I', 'R']), obj_funct)
for rbf in [
'linear', 'cubic', 'thin_plate_spline', 'multiquadric',
'gaussian'
]:
settings = rbfopt.RbfoptSettings(max_evaluations=50, rbf=rbf)
alg = rbfopt.RbfoptAlgorithm(settings, bb)
val, x, itercount, evalcount, fast_evalcount = alg.optimize()
self.assertTrue(evalcount == 50,
msg='Did not use the full evaluation budget' +
' with rbf ' + rbf)
self.assertLessEqual(val,
9.95,
msg='Did not find optimum' + ' with rbf ' +
rbf)
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
file_csv.writerow(csv_header)
hyp_opt = "RBF"
var_lower = [
hyp_domains["learning_rate"][0], hyp_domains["weight_decay"][0]
]
var_upper = [
hyp_domains["learning_rate"][1], hyp_domains["weight_decay"][1]
]
bb = rbfopt.RbfoptUserBlackBox(2, var_lower, var_upper, ['R', 'R'],
evaluate_RBF)
settings = rbfopt.RbfoptSettings(max_evaluations=num_evaluations,
target_objval=0.0)
alg = rbfopt.RbfoptAlgorithm(settings, bb)
val, x, itercount, evalcount, fast_evalcount = alg.optimize()
print("Results with RBF optimizer: " + str({
"target": val,
"learning_rate": x[0],
"weight_decay": x[1]
}) + "\n")
with open(csv_evaluations_file, mode='a') as file_csv:
file_csv = csv.writer(file_csv,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
file_csv.writerow(['', '', '', '', '', '', '', ''])
''' Bayesian hyperparameters optimization '''
print("Beginning hyperparameters optimization with BAY")
with open(csv_evaluations_file, mode='a') as file_csv:
def ems(riperes, sim, lb, ub, nspec, **kwargs):
# This subroutine performs error maximization sampling
# Inputs:
# riperes - results from ripemodel()
# sim - Original black-box simulator (callable in python)
# lb/ub - bounds for each independent variable
# nspec - number of species (can be different than #lb/ub)
# Outputs:
# x - next best input point
# errs - absolute errors obtained on predicted point
# Non-default options can be provided explicitly through kwargs
if "sharedata" in kwargs.keys():
sharedata = kwargs["sharedata"]
else:
sharedata = ripe.sharedata
# poskeys = sharedata["ivars"] + ["x"]
inkeys = list(set(kwargs.keys()) - set(["sharedata"]))
ndim = len(lb)
ns = nspec
# Call atermconstruct.formatinputs to get input data
# in a convenient form
if "x" in inkeys:
conc = kwargs["x"]
dflag = True
else:
conc = [1] * ns
dflag = False
if "frac" in kwargs.keys():
dofrac = True
# conc[i] = kwargs["frac"](conc[i]) # i not defined?
else:
dofrac = False
data, kwargs, fdata, pc, alldata = ripe.atermconstruct.formatinputs(conc, kwargs)
# Handle inkeys exceptions and (possibly) generate a pyomo model
if "Tref" in inkeys:
Tref = sharedata["Tref"]
if "res_sim" in inkeys:
res_sim = kwargs["res_sim"]
else:
if "Tref" in inkeys:
res_sim = constructmodel(riperes, sharedata=sharedata, Tref=Tref)
else:
res_sim = constructmodel(riperes, sharedata=sharedata)
ndim = len(lb)
nd, ns = np.shape(fdata)
if "T" in inkeys:
params = [[], []]
params[0] = riperes["k"]
params[1] = riperes["E"]
else:
params = riperes["k"]
# Check for multiple requested points
# if "nreq" in inkeys:
# nreq = kwargs["nreq"]
# else:
# nreq = 1
# subroutine for using fractional arguments (mole fracs or partial pressure)
def apply_frac(x, doit=True):
if doit:
return kwargs["frac"](x[:])
else:
return x
def check_fun(x):
# Check fun is used to evaluate errors of the current model on provided data
x[:] = apply_frac(x[:], dofrac)
return -1.0 * np.sum(
np.divide(
np.power(np.subtract(x[:ns], res_sim(x[ns:])), 2), riperes["sigma"]
)
)
def max_err(x):
# max_err determines the absolute errors (max called later)
x[:] = apply_frac(x[:], dofrac)
errs = np.absolute(np.subtract(sim(x), res_sim(x)))[0]
return errs
def error_fun(x):
# error fun is the black box objective
x[:] = apply_frac(x[:], dofrac)
sim_conc = sim(x[:])
res_conc = res_sim(x[:])
return -1.0 * np.sum(
np.power(np.divide(np.subtract(sim_conc, res_conc), sim_conc), 2)
)
t_targets = np.zeros([nd, 1])
if dflag:
for i in range(nd):
t_targets[i] = check_fun(alldata[i][:])
bb = rbfopt.RbfoptUserBlackBox(
ndim, np.array(lb), np.array(ub), np.array(["R"] * ndim), error_fun
)
# Request that any point returned be no closer than the other closest point
distance = 10.0
for i in range(nd):
for j in range(nd):
if i != j:
distance = max(
distance,
np.linalg.norm(
alldata[i, ns : ns + ndim] - alldata[j, ns : ns + ndim]
),
)
settings = []
t = datetime.datetime.now()
d_mult = 1.0
settings = rbfopt.RbfoptSettings(
nlp_solver_path=sharedata["nlp_path"],
minlp_solver_path=sharedata["minlp_path"],
print_solver_output=False,
min_dist=d_mult * distance,
algorithm="Gutmann",
rand_seed=int(time.mktime(t.timetuple())),
) # random seed required for RBFopt functionality
# Alternative settings used in testing saved for posterity
# settings = rbfopt.RbfoptSettings( do_infstep=True,nlp_solver_path='/usr/local/ipopt/3.10.2/ipopt/ipopt.pc', minlp_solver_path='~/baron/baron',print_solver_output=False, algorithm='MSRM',global_search_method = 'solver',rand_seed = int(time.mktime(t.timetuple())), min_dist = d_mult*distance)
# ),#init_strategy='all_corners',
# num_global_searches = 0,)
# provide initialization data if flag is tripped
if not dflag:
# print 'check conc : ',alldata[:,ns:ns+ndim],t_targets
alg = rbfopt.RbfoptAlgorithm(
settings,
bb,
do_init_strategy=False,
init_node_pos=alldata[:, ns : ns + ndim],
init_node_val=t_targets,
) # ,num_nodes_at_restart=nd)
else:
alg = rbfopt.RbfoptAlgorithm(settings, bb)
# Call to rbfopt in a manner that terminal is not polluted
f = open(os.devnull, "w")
alg.set_output_stream(f)
val, new_x, itercount, evalcount, fast_evalcount = alg.optimize(pause_after_iters=1)
f.close()
if len(new_x) < ndim:
x = list(new_x) + [0] * (ndim - len(new_x))
else:
x = list(new_x)
# Calculate relevant metrics and identify output responsible for error violation
errs = max_err(x)
# maxv = np.max(errs)
# loc = np.argmax(errs)
if "T" in inkeys:
# prop_x = x[:-1]
prop_t = x[-1]
# else:
# prop_x = x
# print 'Maximum error on new proposed point : ', maxv,' on species # ',loc+1
# print 'Proposed initial concentrations', prop_x
if "T" in inkeys:
print("Proposed Temperature", prop_t)
return [x, errs]
def run_alg():
bb = rbfopt.RbfoptUserBlackBox(dimension=dimension, var_lower=var_lower,
var_upper=var_upper, var_type=var_type, obj_funct=wrapper)
settings = rbfopt.RbfoptSettings(max_evaluations=evaluations)
alg = rbfopt.RbfoptAlgorithm(settings, bb)
return alg.optimize()
def train_rbfopt(self):
with tf.Graph().as_default() as self.g:
with slim.arg_scope(
[slim.conv2d, slim.fully_connected],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(
0.0, self.init_variance),
weights_regularizer=slim.l2_regularizer(0.00001)):
self.x = tf.placeholder(tf.float32, (self.n, ))
self.labels = tf.placeholder(tf.float32,
(self.OUT_SIZE, self.OUT_SIZE, 3))
self.y = tf.placeholder(tf.float32,
(self.OUT_SIZE, self.OUT_SIZE, 3))
self.loss = tf.reduce_mean(
tf.sqrt(tf.nn.l2_loss(self.y - self.labels)))
self.optimizer = tf.train.GradientDescentOptimizer(
learning_rate=self.learning_rate)
self.loss_summary = tf.summary.scalar('losses/main_loss',
self.loss)
def obj_funct(x):
self.X = x
self.Y = black_box(self.X,
output_size=self.OUT_SIZE,
global_step=0,
frames_path=self.frames_path)
self.current_fitness, self.new_summary = self.sess.run(
[self.loss, self.loss_summary],
feed_dict={
self.x: self.X,
self.labels: self.fake_target,
self.y: self.Y
})
self.writer.add_summary(self.new_summary, self.global_step)
print("step number : %2d Loss: %4.2f \n" %
(self.global_step, self.current_fitness))
self.all_inputs.append(self.X)
self.all_losses.append(self.current_fitness)
if self.ALLOW_LOGGING and (self.global_step % self.bb_log_frq
== 0):
scipy.misc.imsave(
os.path.join(self.frames_log_dir,
str(self.global_step) + ".jpg"),
inverse_transform(self.Y))
self.global_step += 1
return np.copy(self.current_fitness)
with tf.Session(graph=self.g,
config=tf.ConfigProto(
gpu_options=self.gpu_options)) as self.sess:
self.writer = tf.summary.FileWriter(self.train_log_dir,
self.sess.graph)
tf.global_variables_initializer().run()
self.all_inputs = []
self.all_losses = []
self.global_step = 0
self.start_time = time.time()
bb = rbfopt.RbfoptUserBlackBox(self.n, -np.ones(self.n),
np.ones(self.n),
np.array(['R'] * self.n), obj_funct)
settings = rbfopt.RbfoptSettings(max_evaluations=self.STEPS_NUMBER,
global_search_method="sampling",
algorithm="Gutmann")
alg = rbfopt.RbfoptAlgorithm(settings, bb)
val, x, itercount, evalcount, fast_evalcount = alg.optimize()
def rbfopt(self, x0, bnds, options
): # noisy, cheap function option. supports discrete variables
# https://rbfopt.readthedocs.io/en/latest/rbfopt_user_black_box.html
# https://rbfopt.readthedocs.io/en/latest/rbfopt_settings.html
# https://rbfopt.readthedocs.io/en/latest/rbfopt_algorithm.html
timer_start = timer()
if options['stop_criteria_type'] == 'Iteration Maximum':
max_eval = int(options['stop_criteria_val'])
max_time = 1E30
elif options['stop_criteria_type'] == 'Maximum Time [min]':
max_eval = 10000 # will need to check if rbfopt changes based on iteration
max_time = options['stop_criteria_val'] * 60
var_type = ['R'] * np.size(
x0) # specifies that all variables are continious
output = {'success': False, 'message': []}
# Intialize and report any problems to log, not to console window
stdout = io.StringIO()
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr):
with contextlib.redirect_stdout(stdout):
bb = rbfopt.RbfoptUserBlackBox(np.size(x0), np.array(bnds[0]),
np.array(bnds[1]),
np.array(var_type),
self.obj_fcn)
settings = rbfopt.RbfoptSettings(
max_iterations=max_eval,
max_evaluations=max_eval,
max_cycles=1E30,
max_clock_time=max_time,
minlp_solver_path=path['bonmin'],
nlp_solver_path=path['ipopt'])
algo = rbfopt.RbfoptAlgorithm(settings, bb, init_node_pos=x0)
val, x, itercount, evalcount, fast_evalcount = algo.optimize()
obj_fcn, x, shock_output = self.Scaled_Fit_Fun(
x, optimizing=False)
output['message'] = 'Optimization terminated successfully.'
output['success'] = True
ct_out = stdout.getvalue()
ct_err = stderr.getvalue()
print(ct_out)
# opt.last_optimum_value() is the same as optimal obj_fcn
res = {
'x': x,
'shock': shock_output,
'fval': obj_fcn,
'nfev': evalcount + fast_evalcount,
'success': output['success'],
'message': output['message'],
'time': timer() - timer_start
}
return res
