def main():
(machine, processor, nodes, cores) = GetMachineConfiguration()
print("machine: " + machine + " processor: " + processor + " num_nodes: " +
str(nodes) + " num_cores: " + str(cores))
# Parse command line arguments
args = parse_args()
bmin = args.bmin
device = args.device
bmax = args.bmax
eta = args.eta
nrun = args.nrun
npernode = args.npernode
ntask = args.ntask
Nloop = args.Nloop
restart = args.restart
TUNER_NAME = args.optimization
ot.RandomGenerator.SetSeed(args.seed)
TLA = False
print(args)
print("machine: " + machine + " processor: " + processor + " num_nodes: " +
str(nodes) + " num_cores: " + str(cores))
os.environ['MACHINE_NAME'] = machine
os.environ['TUNER_NAME'] = TUNER_NAME
dataset = Categoricalnorm(['cora', 'citeseer'],
transform="onehot",
name="dataset")
lr = Real(1e-5, 1e-2, name="lr")
hidden = Integer(4, 64, transform="normalize", name="hidden")
weight_decay = Real(1e-5, 1e-2, name="weight_decay")
dropout = Real(0.1, 0.9, name="dropout")
validation_loss = Real(0., 1., name="validation_loss")
IS = Space([dataset])
PS = Space([weight_decay, hidden, lr, dropout])
OS = Space([validation_loss])
constraints = {}
constants = {
"nodes": nodes,
"cores": cores,
"npernode": npernode,
"bmin": bmin,
"bmax": bmax,
"eta": eta,
"device": device
}
print(IS, PS, OS, constraints)
problem = TuningProblem(IS,
PS,
OS,
objectives,
constraints,
constants=constants)
computer = Computer(nodes=nodes, cores=cores, hosts=None)
options = Options()
options['model_processes'] = 4 # parallel cholesky for each LCM kernel
# options['model_threads'] = 1
# options['model_restarts'] = args.Nrestarts
# options['distributed_memory_parallelism'] = False
# parallel model restart
options['model_restarts'] = restart
options['distributed_memory_parallelism'] = False
options['shared_memory_parallelism'] = False
# options['mpi_comm'] = None
options['model_class'] = 'Model_LCM' # Model_GPy_LCM or Model_LCM
options['verbose'] = False
options['sample_class'] = 'SampleOpenTURNS'
options['budget_min'] = bmin
options['budget_max'] = bmax
options['budget_base'] = eta
smax = int(
np.floor(
np.log(options['budget_max'] / options['budget_min']) /
np.log(options['budget_base'])))
budgets = [
options['budget_max'] / options['budget_base']**x
for x in range(smax + 1)
]
NSs = [
int((smax + 1) / (s + 1)) * options['budget_base']**s
for s in range(smax + 1)
]
NSs_all = NSs.copy()
budget_all = budgets.copy()
for s in range(smax + 1):
for n in range(s):
NSs_all.append(int(NSs[s] / options['budget_base']**(n + 1)))
budget_all.append(int(budgets[s] *
options['budget_base']**(n + 1)))
Ntotal = int(sum(NSs_all) * Nloop)
Btotal = int(
np.dot(np.array(NSs_all), np.array(budget_all)) /
options['budget_max'] * Nloop
) # total number of evaluations at highest budget -- used for single-fidelity tuners
print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}")
print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all)
print("total number of samples: ", Ntotal)
print("total number of evaluations at highest budget: ", Btotal)
print()
options.validate(computer=computer)
data = Data(problem)
# giventask = [[0.2, 0.5]]
giventask = []
dataset_list = args.dataset.split('-')
for dataset in dataset_list:
giventask.append([dataset])
NI = len(giventask)
assert NI == ntask # make sure number of tasks match
if (TUNER_NAME == 'GPTune'):
gt = GPTune(problem,
computer=computer,
data=data,
options=options,
driverabspath=os.path.abspath(__file__))
""" Building MLA with the given list of tasks """
NS = Btotal
if args.nrun > 0:
NS = args.nrun
NS1 = max(NS // 2, 1)
(data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(
f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt",
"a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" dataset = {data.I[tid][0]}")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ',
min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" dataset = {data.I[tid][0]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
if (TUNER_NAME == 'opentuner'):
NS = Btotal
(data, stats) = OpenTuner(T=giventask,
NS=NS,
tp=problem,
computer=computer,
run_id="OpenTuner",
niter=1,
technique=None)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(
f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt",
"a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" dataset = {data.I[tid][0]}")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])],
'Oopt ',
min(data.O[tid][:NS])[0], 'nth ',
np.argmin(data.O[tid][:NS]))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" dataset = {data.I[tid][0]}\n")
# results_file.write(f" Ps {data.P[tid][:NS]}\n")
results_file.write(f" Os {data.O[tid][:NS].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
# single-fidelity version of hpbandster
if (TUNER_NAME == 'TPE'):
NS = Btotal
(data, stats) = HpBandSter(T=giventask,
NS=NS,
tp=problem,
computer=computer,
options=options,
run_id="HpBandSter",
niter=1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(
f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt",
"a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" dataset = {data.I[tid][0]}")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ',
min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" dataset = {data.I[tid][0]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
if (TUNER_NAME == 'GPTuneBand'):
data = Data(problem)
gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options)
(data, stats, data_hist) = gt.MB_LCM(NS=Nloop, Igiven=giventask)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(
f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt",
"a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" dataset = {data.I[tid][0]}")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
nth = np.argmin(data.O[tid])
Popt = data.P[tid][nth]
# find which arm and which sample the optimal param is from
for arm in range(len(data_hist.P)):
try:
idx = (data_hist.P[arm]).index(Popt)
arm_opt = arm
except ValueError:
pass
print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth)
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" dataset = {data.I[tid][0]}\n")
# results_file.write(f" Ps {data.P[tid]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
# multi-fidelity version
if (TUNER_NAME == 'hpbandster'):
NS = Ntotal
(data, stats) = HpBandSter_bandit(T=giventask,
NS=NS,
tp=problem,
computer=computer,
options=options,
run_id="hpbandster_bandit",
niter=1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(
f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt",
"a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" dataset = {data.I[tid][0]}")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
# print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
max_budget = 0.
Oopt = 99999
Popt = None
nth = None
for idx, (config,
out) in enumerate(zip(data.P[tid],
data.O[tid].tolist())):
for subout in out[0]:
budget_cur = subout[0]
if budget_cur > max_budget:
max_budget = budget_cur
Oopt = subout[1]
Popt = config
nth = idx
elif budget_cur == max_budget:
if subout[1] < Oopt:
Oopt = subout[1]
Popt = config
nth = idx
print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth)
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" dataset = {data.I[tid][0]}\n")
# results_file.write(f" Ps {data.P[tid]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
def main():
import matplotlib.pyplot as plt
args = parse_args()
ntask = args.ntask
nruns = args.nruns
TUNER_NAME = args.optimization
Nloop = args.Nloop
plot = args.plot
expid = args.expid
restart = args.restart
(machine, processor, nodes, cores) = GetMachineConfiguration()
print("machine: " + machine + " processor: " + processor + " num_nodes: " +
str(nodes) + " num_cores: " + str(cores))
os.environ['MACHINE_NAME'] = machine
os.environ['TUNER_NAME'] = TUNER_NAME
input_space = Space([Real(0., 10., transform="normalize", name="t")])
parameter_space = Space([Real(0., 1., transform="normalize", name="x")])
# input_space = Space([Real(0., 0.0001, "uniform", "normalize", name="t")])
# parameter_space = Space([Real(-1., 1., "uniform", "normalize", name="x")])
output_space = Space([Real(float('-Inf'), float('Inf'), name="y")])
constraints = {"cst1": "x >= 0. and x <= 1."}
# problem = TuningProblem(input_space, parameter_space,output_space, objectives, constraints, models) # with performance model
problem = TuningProblem(input_space, parameter_space, output_space,
objectives, constraints,
None) # no performance model
computer = Computer(nodes=nodes, cores=cores, hosts=None)
options = Options()
options['model_restarts'] = restart
options['distributed_memory_parallelism'] = False
options['shared_memory_parallelism'] = False
options['objective_evaluation_parallelism'] = False
options['objective_multisample_threads'] = 1
options['objective_multisample_processes'] = 1
options['objective_nprocmax'] = 1
options['model_processes'] = 1
# options['model_threads'] = 1
# options['model_restart_processes'] = 1
# options['search_multitask_processes'] = 1
# options['search_multitask_threads'] = 1
# options['search_threads'] = 16
# options['mpi_comm'] = None
#options['mpi_comm'] = mpi4py.MPI.COMM_WORLD
options['model_class'] = 'Model_GPy_LCM' #'Model_LCM'
options['verbose'] = False
# options['sample_algo'] = 'MCS'
# options['sample_class'] = 'SampleLHSMDU'
options.validate(computer=computer)
options['budget_min'] = bmin
options['budget_max'] = bmax
options['budget_base'] = eta
smax = int(
np.floor(
np.log(options['budget_max'] / options['budget_min']) /
np.log(options['budget_base'])))
budgets = [
options['budget_max'] / options['budget_base']**x
for x in range(smax + 1)
]
NSs = [
int((smax + 1) / (s + 1)) * options['budget_base']**s
for s in range(smax + 1)
]
NSs_all = NSs.copy()
budget_all = budgets.copy()
for s in range(smax + 1):
for n in range(s):
NSs_all.append(int(NSs[s] / options['budget_base']**(n + 1)))
budget_all.append(int(budgets[s] *
options['budget_base']**(n + 1)))
Ntotal = int(sum(NSs_all) * Nloop)
Btotal = int(
np.dot(np.array(NSs_all), np.array(budget_all)) / options['budget_max']
) # total number of evaluations at highest budget -- used for single-fidelity tuners
print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all)
print("total number of samples: ", Ntotal)
print("total number of evaluations at highest budget: ", Btotal)
print(f"Sampler: {options['sample_class']}, {options['sample_algo']}")
print()
data = Data(problem)
# giventask = [[1.0], [5.0], [10.0]]
# giventask = [[1.0], [1.2], [1.3]]
# giventask = [[1.0]]
# t_end = args.t_end
giventask = [[i] for i in np.arange(1, ntask / 2 + 1, 0.5).tolist()]
# giventask = [[i] for i in np.arange(1, 1.5, 0.05).tolist()]
# giventask = [[1.0], [1.05], [1.1]]
NI = len(giventask)
assert NI == ntask # make sure number of tasks match
np.set_printoptions(suppress=False, precision=3)
if (TUNER_NAME == 'GPTuneBand'):
NS = Nloop
data = Data(problem)
gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options)
(data, stats, data_hist) = gt.MB_LCM(NS=Nloop, Igiven=giventask)
print("Tuner: ", TUNER_NAME)
print("Sampler class: ", options['sample_class'])
print("Model class: ", options['model_class'])
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" t = {data.I[tid][0]:.2f}")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
nth = np.argmin(data.O[tid])
Popt = data.P[tid][nth]
# find which arm and which sample the optimal param is from
for arm in range(len(data_hist.P)):
try:
idx = (data_hist.P[arm]).index(Popt)
arm_opt = arm
except ValueError:
pass
print(' Popt ', Popt, 'Oopt ',
min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ',
(arm_opt, idx))
if (TUNER_NAME == 'GPTune'):
NS = Btotal
if args.nruns > 0:
NS = args.nruns
print("In GPTune, using the given number of nruns ", NS)
NS1 = max(NS // 2, 1)
gt = GPTune(problem,
computer=computer,
data=data,
options=options,
driverabspath=os.path.abspath(__file__))
""" Building MLA with the given list of tasks """
(data, modeler, stats) = gt.MLA(NS=NS,
NI=NI,
Igiven=giventask,
NS1=NS1)
print("stats: ", stats)
print("model class: ", options['model_class'])
print("Model restart: ", restart)
""" Print all input and parameter samples """
sum_Oopt = 0.
for tid in range(NI):
print("tid: %d" % (tid))
print(f" t: {data.I[tid][0]:.2f} ")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])],
f'Oopt {min(data.O[tid])[0]:.3f}', 'nth ',
np.argmin(data.O[tid]))
sum_Oopt += min(data.O[tid])[0]
# print("sum of all optimal objectives", sum_Oopt)
if (TUNER_NAME == 'opentuner'):
NS = Btotal
(data, stats) = OpenTuner(T=giventask,
NS=NS,
tp=problem,
computer=computer,
run_id="OpenTuner",
niter=1,
technique=None)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" t: {data.I[tid][0]:.2f} ")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])],
'Oopt ',
min(data.O[tid][:NS])[0], 'nth ',
np.argmin(data.O[tid][:NS]))
# single fidelity version of hpbandster
if (TUNER_NAME == 'TPE'):
NS = Btotal
(data, stats) = HpBandSter(T=giventask,
NS=NS,
tp=problem,
computer=computer,
run_id="HpBandSter",
niter=1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" t: {data.I[tid][0]:.2f} ")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ',
min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
# multi-fidelity version
if (TUNER_NAME == 'hpbandster'):
NS = Ntotal
(data, stats) = HpBandSter_bandit(T=giventask,
NS=NS,
tp=problem,
computer=computer,
options=options,
run_id="hpbandster_bandit",
niter=1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" t: {data.I[tid][0]:.2f} ")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
# print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
max_budget = 0.
Oopt = 99999
Popt = None
nth = None
for idx, (config,
out) in enumerate(zip(data.P[tid],
data.O[tid].tolist())):
for subout in out[0]:
budget_cur = subout[0]
if budget_cur > max_budget:
max_budget = budget_cur
Oopt = subout[1]
Popt = config
nth = idx
elif budget_cur == max_budget:
if subout[1] < Oopt:
Oopt = subout[1]
Popt = config
nth = idx
print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth)
if plot == 1:
x = np.arange(0., 1., 0.0001)
ymean_set = [] # stores predicted function values
ytrue_set = []
for tid in range(len(data.I)):
p = data.I[tid]
t = p[0]
fig = plt.figure(figsize=[12.8, 9.6])
I_orig = p
kwargst = {
input_space[k].name: I_orig[k]
for k in range(len(input_space))
}
y = np.zeros([len(x), 1])
y_mean = np.zeros([len(x)])
y_std = np.zeros([len(x)])
for i in range(len(x)):
P_orig = [x[i]]
kwargs = {
parameter_space[k].name: P_orig[k]
for k in range(len(parameter_space))
}
kwargs.update(kwargst)
y[i] = objectives(kwargs)
if (TUNER_NAME == 'GPTune'):
(y_mean[i], var) = predict_aug(modeler, gt, kwargs, tid)
y_std[i] = np.sqrt(var)
# print(y_mean[i],y_std[i],y[i])
fontsize = 40
plt.rcParams.update({'font.size': 40})
plt.plot(x, y, 'b', lw=2, label='true')
plt.plot(x, y_mean, 'k', lw=3, zorder=9, label='prediction')
plt.fill_between(x,
y_mean - y_std,
y_mean + y_std,
alpha=0.2,
color='k')
plt.ylim(0, 2)
# print(data.P[tid])
plt.scatter(data.P[tid],
data.O[tid],
c='r',
s=50,
zorder=10,
edgecolors=(0, 0, 0),
label='sample')
plt.xlabel('x', fontsize=fontsize + 2)
plt.ylabel('y(t,x)', fontsize=fontsize + 2)
plt.title('t=%f' % t, fontsize=fontsize + 2)
print('t:', t, 'x:', x[np.argmin(y)], 'ymin:', y.min())
# legend = plt.legend(loc='upper center', shadow=True, fontsize='x-large')
legend = plt.legend(loc='upper right',
shadow=False,
fontsize=fontsize)
annot_min(x, y)
# plt.show()
plt.show(block=False)
plt.pause(0.5)
# input("Press [enter] to continue.")
# fig.savefig('obj_t_%f.eps'%t)
fig.savefig(f'obj_ntask{NI}_{expid}_tid_{tid}_t_{t:.1f}.pdf')
ymean_set.append(y_mean)
ytrue_set.append(y)
# show the distance among surrogate functions
R = np.zeros(
(NI,
NI)) # Pearson sample correlation matrix of learned surrogates
R_true = np.zeros(
(NI, NI)) # Pearson sample correlation of true functions
for i in range(NI):
for ip in range(i, NI):
ymean_i = ymean_set[i]
ymean_ip = ymean_set[ip]
ytrue_i = np.array((ytrue_set[i]).reshape((1, -1)))[0]
ytrue_ip = np.array((ytrue_set[ip]).reshape((1, -1)))[0]
# find the Pearson sample correlation coefficient
R[i, ip], _ = scipy.stats.pearsonr(ymean_i, ymean_ip)
R_true[i, ip], _ = scipy.stats.pearsonr(ytrue_i, ytrue_ip)
print("The correlation matrix among surrogate functions is: \n", R)
print("The correlation matrix among true functions is: \n", R_true)
new_Rtrue = R_true[np.triu_indices(R_true.shape[0], 1)]
new_R = R[np.triu_indices(R.shape[0], 1)]
print("The mean absolute error is: \n",
np.mean(abs(new_Rtrue - new_R)))
print("The mean relative error is: \n",
np.mean(abs(new_Rtrue - new_R) / abs(new_R)))
def main():
(machine, processor, nodes, cores) = GetMachineConfiguration()
print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores))
# Parse command line arguments
args = parse_args()
bmin = args.bmin
device = args.device
bmax = args.bmax
eta = args.eta
nrun = args.nrun
npernode = args.npernode
ntask = args.ntask
Nloop = args.Nloop
restart = args.restart
TUNER_NAME = args.optimization
TLA = False
(machine, processor, nodes, cores) = GetMachineConfiguration()
print(args)
print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores))
os.environ['MACHINE_NAME'] = machine
os.environ['TUNER_NAME'] = TUNER_NAME
# os.system("mkdir -p scalapack-driver/bin/%s; cp ../build/pdqrdriver scalapack-driver/bin/%s/.;" %(machine, machine))
ntrain = Integer(1000, 10000, transform="normalize", name="ntrain")
nvalid = Integer(256, 2048, transform="normalize", name="nvalid")
lr = Real(1e-6, 1e-2, name="lr")
optimizer = Categoricalnorm(['Adam', 'SGD'], transform="onehot", name="optimizer")
sgd_momentum = Real(0, 0.99, name="sgd_momentum")
num_conv_layers = Integer(1, 3, transform="normalize", name="num_conv_layers")
num_filters_1 = Integer(4, 64, transform="normalize", name="num_filters_1")
num_filters_2 = Integer(4, 64, transform="normalize", name="num_filters_2")
num_filters_3 = Integer(4, 64, transform="normalize", name="num_filters_3")
dropout_rate = Real(0, 0.9, name="dropout_rate")
num_fc_units = Integer(8, 256, transform="normalize", name="num_fc_units")
validation_loss = Real(float("-Inf"), float("Inf"), name="validation_loss")
IS = Space([ntrain, nvalid])
PS = Space([lr, optimizer, sgd_momentum, num_conv_layers, num_filters_1, num_filters_2, num_filters_3, dropout_rate, num_fc_units])
OS = Space([validation_loss])
constraints = {}
constants={"nodes":nodes,"cores":cores,"npernode":npernode,"bmin":bmin,"bmax":bmax,"eta":eta, "device":device}
print(IS, PS, OS, constraints)
problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants)
computer = Computer(nodes=nodes, cores=cores, hosts=None)
options = Options()
options['model_processes'] = 4 # parallel cholesky for each LCM kernel
# options['model_threads'] = 1
# options['model_restarts'] = args.Nrestarts
# options['distributed_memory_parallelism'] = False
# parallel model restart
options['model_restarts'] = restart
options['distributed_memory_parallelism'] = False
options['shared_memory_parallelism'] = False
# options['mpi_comm'] = None
options['model_class'] = 'Model_LCM' # Model_GPy_LCM or Model_LCM
options['verbose'] = False
options['budget_min'] = bmin
options['budget_max'] = bmax
options['budget_base'] = eta
smax = int(np.floor(np.log(options['budget_max']/options['budget_min'])/np.log(options['budget_base'])))
budgets = [options['budget_max'] /options['budget_base']**x for x in range(smax+1)]
NSs = [int((smax+1)/(s+1))*options['budget_base']**s for s in range(smax+1)]
NSs_all = NSs.copy()
budget_all = budgets.copy()
for s in range(smax+1):
for n in range(s):
NSs_all.append(int(NSs[s]/options['budget_base']**(n+1)))
budget_all.append(int(budgets[s]*options['budget_base']**(n+1)))
Ntotal = int(sum(NSs_all) * Nloop)
Btotal = int(np.dot(np.array(NSs_all), np.array(budget_all))/options['budget_max'] * Nloop) # total number of evaluations at highest budget -- used for single-fidelity tuners
print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}")
print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all)
print("total number of samples: ", Ntotal)
print("total number of evaluations at highest budget: ", Btotal)
print()
options.validate(computer = computer)
data = Data(problem)
# giventask = [[0.2, 0.5]]
if ntask == 1:
giventask = [[args.ntrain, args.nvalid]]
# giventask = [[3000, 1000]]
NI=len(giventask)
assert NI == ntask # make sure number of tasks match
if(TUNER_NAME=='GPTune'):
gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__))
""" Building MLA with the given list of tasks """
NS = Btotal
if args.nrun > 0:
NS = args.nrun
NS1 = max(NS//2, 1)
(data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
if(TUNER_NAME=='opentuner'):
NS = Btotal
(data,stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS]))
# single-fidelity version of hpbandster
if(TUNER_NAME=='TPE'):
NS = Btotal
(data,stats)=HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, run_id="HpBandSter", niter=1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
if(TUNER_NAME=='GPTuneBand'):
data = Data(problem)
gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options)
(data, stats, data_hist)=gt.MB_LCM(NS = Nloop, Igiven = giventask)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
nth = np.argmin(data.O[tid])
Popt = data.P[tid][nth]
# find which arm and which sample the optimal param is from
for arm in range(len(data_hist.P)):
try:
idx = (data_hist.P[arm]).index(Popt)
arm_opt = arm
except ValueError:
pass
print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx))
# multi-fidelity version
if(TUNER_NAME=='hpbandster'):
NS = Ntotal
(data,stats)=HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
# print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
max_budget = 0.
Oopt = 99999
Popt = None
nth = None
for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())):
for subout in out[0]:
budget_cur = subout[0]
if budget_cur > max_budget:
max_budget = budget_cur
Oopt = subout[1]
Popt = config
nth = idx
elif budget_cur == max_budget:
if subout[1] < Oopt:
Oopt = subout[1]
Popt = config
nth = idx
print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth)
def main():
(machine, processor, nodes, cores) = GetMachineConfiguration()
print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores))
# Parse command line arguments
args = parse_args()
bmin = args.bmin
bmax = args.bmax
eta = args.eta
amin = args.amin
amax = args.amax
cmin = args.cmin
cmax = args.cmax
nprocmin_pernode = args.nprocmin_pernode
ntask = args.ntask
Nloop = args.Nloop
restart = args.restart
TUNER_NAME = args.optimization
TLA = False
os.environ['MACHINE_NAME'] = machine
os.environ['TUNER_NAME'] = TUNER_NAME
# os.system("mkdir -p scalapack-driver/bin/%s; cp ../build/pdqrdriver scalapack-driver/bin/%s/.;" %(machine, machine))
nprocmax = nodes*cores
nprocmin = nodes*nprocmin_pernode
a_val = Real(amin, amax, transform="normalize", name="a_val")
c_val = Real(cmin, cmax, transform="normalize", name="c_val")
Px = Integer(1, nprocmax, transform="normalize", name="Px")
Py = Integer(1, nprocmax, transform="normalize", name="Py")
Nproc = Integer(nprocmin, nprocmax, transform="normalize", name="Nproc")
strong_threshold = Real(0, 1, transform="normalize", name="strong_threshold")
trunc_factor = Real(0, 0.999, transform="normalize", name="trunc_factor")
P_max_elmts = Integer(1, 12, transform="normalize", name="P_max_elmts")
# coarsen_type = Categoricalnorm (['0', '1', '2', '3', '4', '6', '8', '10'], transform="onehot", name="coarsen_type")
coarsen_type = Categoricalnorm (['0', '1', '2', '3', '4', '8', '10'], transform="onehot", name="coarsen_type")
relax_type = Categoricalnorm (['-1', '0', '6', '8', '16', '18'], transform="onehot", name="relax_type")
smooth_type = Categoricalnorm (['5', '6', '8', '9'], transform="onehot", name="smooth_type")
smooth_num_levels = Integer(0, 5, transform="normalize", name="smooth_num_levels")
interp_type = Categoricalnorm (['0', '3', '4', '5', '6', '8', '12'], transform="onehot", name="interp_type")
agg_num_levels = Integer(0, 5, transform="normalize", name="agg_num_levels")
r = Real(float("-Inf"), float("Inf"), name="r")
IS = Space([a_val, c_val])
PS = Space([Px, Py, Nproc, strong_threshold, trunc_factor, P_max_elmts, coarsen_type, relax_type, smooth_type, smooth_num_levels, interp_type, agg_num_levels])
OS = Space([r])
cst1 = f"Px * Py <= Nproc"
cst2 = f"not(P_max_elmts==10 and coarsen_type=='6' and relax_type=='18' and smooth_type=='6' and smooth_num_levels==3 and interp_type=='8' and agg_num_levels==1)"
constraints = {"cst1": cst1,"cst2": cst2}
constants={"nodes":nodes,"cores":cores,"bmin":bmin,"bmax":bmax,"eta":eta}
print(IS, PS, OS, constraints)
problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants)
computer = Computer(nodes=nodes, cores=cores, hosts=None)
options = Options()
options['model_processes'] = 4 # parallel cholesky for each LCM kernel
# options['model_threads'] = 1
# options['model_restarts'] = args.Nrestarts
# options['distributed_memory_parallelism'] = False
# parallel model restart
options['model_restarts'] = restart
options['distributed_memory_parallelism'] = False
options['shared_memory_parallelism'] = False
# options['mpi_comm'] = None
options['model_class'] = 'Model_LCM' # Model_GPy_LCM or Model_LCM
options['verbose'] = False
# choose sampler
# options['sample_class'] = 'SampleOpenTURNS'
if args.lhs == 1:
options['sample_class'] = 'SampleLHSMDU'
options['sample_algo'] = 'LHS-MDU'
options.validate(computer=computer)
options['budget_min'] = bmin
options['budget_max'] = bmax
options['budget_base'] = eta
smax = int(np.floor(np.log(options['budget_max']/options['budget_min'])/np.log(options['budget_base'])))
budgets = [options['budget_max'] /options['budget_base']**x for x in range(smax+1)]
NSs = [int((smax+1)/(s+1))*options['budget_base']**s for s in range(smax+1)]
NSs_all = NSs.copy()
budget_all = budgets.copy()
for s in range(smax+1):
for n in range(s):
NSs_all.append(int(NSs[s]/options['budget_base']**(n+1)))
budget_all.append(int(budgets[s]*options['budget_base']**(n+1)))
Ntotal = int(sum(NSs_all) * Nloop)
Btotal = int(np.dot(np.array(NSs_all), np.array(budget_all))/options['budget_max'] * Nloop) # total number of evaluations at highest budget -- used for single-fidelity tuners
print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}")
print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all)
print("total number of samples: ", Ntotal)
print("total number of evaluations at highest budget: ", Btotal)
print()
data = Data(problem)
giventask = [[(amax-amin)*random()+amin,(cmax-cmin)*random()+cmin] for i in range(ntask)]
# giventask = [[0.2, 0.5]]
if ntask == 1:
giventask = [[args.a, args.c]]
NI=len(giventask)
assert NI == ntask # make sure number of tasks match
# # the following will use only task lists stored in the pickle file
# data = Data(problem)
if(TUNER_NAME=='GPTune'):
gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__))
""" Building MLA with the given list of tasks """
NS = Btotal
if args.nrun > 0:
NS = args.nrun
NS1 = max(NS//2, 1)
(data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
if TLA is True:
""" Call TLA for 2 new tasks using the constructed LCM model"""
newtask = [[0.5, 0.3], [0.2, 1.0]]
(aprxopts, objval, stats) = gt.TLA1(newtask, NS=None)
print("stats: ", stats)
""" Print the optimal parameters and function evaluations"""
for tid in range(len(newtask)):
print("new task: %s" % (newtask[tid]))
print(' predicted Popt: ', aprxopts[tid], ' objval: ', objval[tid])
if(TUNER_NAME=='opentuner'):
NS = Btotal
(data,stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS]))
# single-fidelity version of hpbandster
if(TUNER_NAME=='TPE'):
NS = Btotal
(data,stats)=HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, run_id="HpBandSter", niter=1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
if(TUNER_NAME=='GPTuneBand'):
data = Data(problem)
gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options)
(data, stats, data_hist)=gt.MB_LCM(NS = Nloop, Igiven = giventask)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
nth = np.argmin(data.O[tid])
Popt = data.P[tid][nth]
# find which arm and which sample the optimal param is from
for arm in range(len(data_hist.P)):
try:
idx = (data_hist.P[arm]).index(Popt)
arm_opt = arm
except ValueError:
pass
print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx))
if(TUNER_NAME=='GPTuneBand_single'):
def merge_dict(mydict, newdict):
for key in mydict.keys():
mydict[key] += newdict[key]
data_all = []
stats_all = {}
for singletask in giventask:
NI = 1
cur_task = [singletask]
data = Data(problem)
gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options)
(data, stats)=gt.MB_LCM(NS = Nloop, Igiven = cur_task)
data_all.append(data)
merge_dict(stats_all, stats)
print("Finish one single task tuning")
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
tid = 0
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
print("Finish tuning...")
print("Tuner: ", TUNER_NAME)
print("stats_all: ", stats_all)
for i in range(len(data_all)):
data = data_all[i]
for tid in range(NI):
print("tid: %d" % (i))
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
# multi-fidelity version
if(TUNER_NAME=='hpbandster'):
NS = Ntotal
(data,stats)=HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1)
print("stats: ", stats)
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]")
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
# print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
max_budget = 0.
Oopt = 99999
Popt = None
nth = None
for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())):
for subout in out[0]:
budget_cur = subout[0]
if budget_cur > max_budget:
max_budget = budget_cur
Oopt = subout[1]
Popt = config
nth = idx
elif budget_cur == max_budget:
if subout[1] < Oopt:
Oopt = subout[1]
Popt = config
nth = idx
print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth)
def main():
# Parse command line arguments
args = parse_args()
# Extract arguments
ntask = args.ntask
npernode = args.npernode
optimization = args.optimization
nrun = args.nrun
bmin = args.bmin
bmax = args.bmax
eta = args.eta
Nloop = args.Nloop
restart = args.restart
expid = args.expid
TUNER_NAME = args.optimization
(machine, processor, nodes, cores) = GetMachineConfiguration()
ot.RandomGenerator.SetSeed(args.seed)
print(args)
print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores))
os.environ['MACHINE_NAME'] = machine
os.environ['TUNER_NAME'] = TUNER_NAME
datasets = args.dataset.split('-')
datafiles = []
giventask = []
assert len(datasets) == ntask
for i in range(len(datasets)):
datafiles.append(f"data/{datasets[i]}")
giventask.append([f"data/{datasets[i]}"])
# datafiles = ["data/branin"]
# Task input parameters
datafile = Categoricalnorm(datafiles, transform="onehot", name="datafile")
# Tuning parameters
h = Real(-10, 10, transform="normalize", name="h")
Lambda = Real(-10, 10, transform="normalize", name="Lambda")
# npernode = Integer(int(math.log2(nprocmin_pernode)), int(math.log2(cores)), transform="normalize", name="npernode")
result = Real(0 , float("Inf"),name="r")
IS = Space([datafile])
# PS = Space([h,Lambda,npernode])
PS = Space([h,Lambda])
OS = Space([result])
constraints = {}
models = {}
constants={"nodes":nodes,"cores":cores,"npernode":npernode,"bmin":bmin,"bmax":bmax,"eta":eta}
""" Print all input and parameter samples """
print(IS, PS, OS, constraints, models)
problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants)
computer = Computer(nodes = nodes, cores = cores, hosts = None)
""" Set and validate options """
options = Options()
options['model_processes'] = 1
# options['model_threads'] = 1
options['model_restarts'] = 1
# options['search_multitask_processes'] = 1
# options['model_restart_processes'] = 1
options['distributed_memory_parallelism'] = False
options['shared_memory_parallelism'] = False
options['model_class '] = 'Model_LCM' # 'Model_GPy_LCM'
options['verbose'] = False
options['sample_class'] = 'SampleOpenTURNS'
options['budget_min'] = bmin
options['budget_max'] = bmax
options['budget_base'] = eta
smax = int(np.floor(np.log(options['budget_max']/options['budget_min'])/np.log(options['budget_base'])))
budgets = [options['budget_max'] /options['budget_base']**x for x in range(smax+1)]
NSs = [int((smax+1)/(s+1))*options['budget_base']**s for s in range(smax+1)]
NSs_all = NSs.copy()
budget_all = budgets.copy()
for s in range(smax+1):
for n in range(s):
NSs_all.append(int(NSs[s]/options['budget_base']**(n+1)))
budget_all.append(int(budgets[s]*options['budget_base']**(n+1)))
Ntotal = int(sum(NSs_all) * Nloop)
Btotal = int(np.dot(np.array(NSs_all), np.array(budget_all))/options['budget_max'] * Nloop) # total number of evaluations at highest budget -- used for single-fidelity tuners
print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}")
print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all)
print("total number of samples: ", Ntotal)
print("total number of evaluations at highest budget: ", Btotal)
print()
options.validate(computer = computer)
# """ Building MLA with the given list of tasks """
data = Data(problem)
NI = ntask
if(TUNER_NAME=='GPTune'):
gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__))
NS = Btotal
if args.nrun > 0:
NS = args.nrun
NS1 = max(NS//2, 1)
(data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d"%(tid))
print(" matrix:%s"%(data.I[tid][0]))
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', -min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" matrix:{data.I[tid][0]:s}\n")
# results_file.write(f" Ps {data.P[tid]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
if(TUNER_NAME=='opentuner'):
NS = Btotal
if args.nrun > 0:
NS = args.nrun
NS1 = max(NS//2, 1)
(data,stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d"%(tid))
print(" matrix:%s"%(data.I[tid][0]))
print(" Ps ", data.P[tid][:NS])
print(" Os ", data.O[tid][:NS])
print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', -min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS]))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" matrix:{data.I[tid][0]:s}\n")
# results_file.write(f" Ps {data.P[tid][:NS]}\n")
results_file.write(f" Os {data.O[tid][:NS].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
if(TUNER_NAME=='TPE'):
NS = Btotal
if args.nrun > 0:
NS = args.nrun
NS1 = max(NS//2, 1)
(data,stats)=HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="HpBandSter", niter=1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d"%(tid))
print(" matrix:%s"%(data.I[tid][0]))
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid])
print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', -min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" matrix:{data.I[tid][0]:s}\n")
# results_file.write(f" Ps {data.P[tid]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
if(TUNER_NAME=='GPTuneBand'):
data = Data(problem)
gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options)
(data, stats, data_hist)=gt.MB_LCM(NS = Nloop, Igiven = giventask)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(" matrix:%s"%(data.I[tid][0]))
print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
nth = np.argmin(data.O[tid])
Popt = data.P[tid][nth]
# find which arm and which sample the optimal param is from
for arm in range(len(data_hist.P)):
try:
idx = (data_hist.P[arm]).index(Popt)
arm_opt = arm
except ValueError:
pass
print(' Popt ', Popt, 'Oopt ', -min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx))
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" matrix:{data.I[tid][0]:s}\n")
# results_file.write(f" Ps {data.P[tid]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()
# multi-fidelity version
if(TUNER_NAME=='hpbandster'):
NS = Ntotal
(data,stats)=HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1)
print("Tuner: ", TUNER_NAME)
print("stats: ", stats)
results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a")
results_file.write(f"Tuner: {TUNER_NAME}\n")
results_file.write(f"stats: {stats}\n")
""" Print all input and parameter samples """
for tid in range(NI):
print("tid: %d" % (tid))
print(" matrix:%s"%(data.I[tid][0]))
# print(" Ps ", data.P[tid])
print(" Os ", data.O[tid].tolist())
# print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
max_budget = 0.
Oopt = 99999
Popt = None
nth = None
for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())):
for subout in out[0]:
budget_cur = subout[0]
if budget_cur > max_budget:
max_budget = budget_cur
Oopt = subout[1]
Popt = config
nth = idx
elif budget_cur == max_budget:
if subout[1] < Oopt:
Oopt = subout[1]
Popt = config
nth = idx
print(' Popt ', Popt, 'Oopt ', -Oopt, 'nth ', nth)
results_file.write(f"tid: {tid:d}\n")
results_file.write(f" matrix:{data.I[tid][0]:s}\n")
# results_file.write(f" Ps {data.P[tid]}\n")
results_file.write(f" Os {data.O[tid].tolist()}\n")
# results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n')
results_file.close()