def main(): (machine, processor, nodes, cores) = GetMachineConfiguration() print("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) # Parse command line arguments args = parse_args() bmin = args.bmin device = args.device bmax = args.bmax eta = args.eta nrun = args.nrun npernode = args.npernode ntask = args.ntask Nloop = args.Nloop restart = args.restart TUNER_NAME = args.optimization ot.RandomGenerator.SetSeed(args.seed) TLA = False print(args) print("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) os.environ['MACHINE_NAME'] = machine os.environ['TUNER_NAME'] = TUNER_NAME dataset = Categoricalnorm(['cora', 'citeseer'], transform="onehot", name="dataset") lr = Real(1e-5, 1e-2, name="lr") hidden = Integer(4, 64, transform="normalize", name="hidden") weight_decay = Real(1e-5, 1e-2, name="weight_decay") dropout = Real(0.1, 0.9, name="dropout") validation_loss = Real(0., 1., name="validation_loss") IS = Space([dataset]) PS = Space([weight_decay, hidden, lr, dropout]) OS = Space([validation_loss]) constraints = {} constants = { "nodes": nodes, "cores": cores, "npernode": npernode, "bmin": bmin, "bmax": bmax, "eta": eta, "device": device } print(IS, PS, OS, constraints) problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants) computer = Computer(nodes=nodes, cores=cores, hosts=None) options = Options() options['model_processes'] = 4 # parallel cholesky for each LCM kernel # options['model_threads'] = 1 # options['model_restarts'] = args.Nrestarts # options['distributed_memory_parallelism'] = False # parallel model restart options['model_restarts'] = restart options['distributed_memory_parallelism'] = False options['shared_memory_parallelism'] = False # options['mpi_comm'] = None options['model_class'] = 'Model_LCM' # Model_GPy_LCM or Model_LCM options['verbose'] = False options['sample_class'] = 'SampleOpenTURNS' options['budget_min'] = bmin options['budget_max'] = bmax options['budget_base'] = eta smax = int( np.floor( np.log(options['budget_max'] / options['budget_min']) / np.log(options['budget_base']))) budgets = [ options['budget_max'] / options['budget_base']**x for x in range(smax + 1) ] NSs = [ int((smax + 1) / (s + 1)) * options['budget_base']**s for s in range(smax + 1) ] NSs_all = NSs.copy() budget_all = budgets.copy() for s in range(smax + 1): for n in range(s): NSs_all.append(int(NSs[s] / options['budget_base']**(n + 1))) budget_all.append(int(budgets[s] * options['budget_base']**(n + 1))) Ntotal = int(sum(NSs_all) * Nloop) Btotal = int( np.dot(np.array(NSs_all), np.array(budget_all)) / options['budget_max'] * Nloop ) # total number of evaluations at highest budget -- used for single-fidelity tuners print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}") print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all) print("total number of samples: ", Ntotal) print("total number of evaluations at highest budget: ", Btotal) print() options.validate(computer=computer) data = Data(problem) # giventask = [[0.2, 0.5]] giventask = [] dataset_list = args.dataset.split('-') for dataset in dataset_list: giventask.append([dataset]) NI = len(giventask) assert NI == ntask # make sure number of tasks match if (TUNER_NAME == 'GPTune'): gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__)) """ Building MLA with the given list of tasks """ NS = Btotal if args.nrun > 0: NS = args.nrun NS1 = max(NS // 2, 1) (data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open( f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" dataset = {data.I[tid][0]}") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) results_file.write(f"tid: {tid:d}\n") results_file.write(f" dataset = {data.I[tid][0]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() if (TUNER_NAME == 'opentuner'): NS = Btotal (data, stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open( f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" dataset = {data.I[tid][0]}") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS])) results_file.write(f"tid: {tid:d}\n") results_file.write(f" dataset = {data.I[tid][0]}\n") # results_file.write(f" Ps {data.P[tid][:NS]}\n") results_file.write(f" Os {data.O[tid][:NS].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() # single-fidelity version of hpbandster if (TUNER_NAME == 'TPE'): NS = Btotal (data, stats) = HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="HpBandSter", niter=1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open( f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" dataset = {data.I[tid][0]}") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) results_file.write(f"tid: {tid:d}\n") results_file.write(f" dataset = {data.I[tid][0]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() if (TUNER_NAME == 'GPTuneBand'): data = Data(problem) gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options) (data, stats, data_hist) = gt.MB_LCM(NS=Nloop, Igiven=giventask) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open( f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" dataset = {data.I[tid][0]}") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) nth = np.argmin(data.O[tid]) Popt = data.P[tid][nth] # find which arm and which sample the optimal param is from for arm in range(len(data_hist.P)): try: idx = (data_hist.P[arm]).index(Popt) arm_opt = arm except ValueError: pass print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth) results_file.write(f"tid: {tid:d}\n") results_file.write(f" dataset = {data.I[tid][0]}\n") # results_file.write(f" Ps {data.P[tid]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() # multi-fidelity version if (TUNER_NAME == 'hpbandster'): NS = Ntotal (data, stats) = HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open( f"GCN_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" dataset = {data.I[tid][0]}") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) # print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) max_budget = 0. Oopt = 99999 Popt = None nth = None for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())): for subout in out[0]: budget_cur = subout[0] if budget_cur > max_budget: max_budget = budget_cur Oopt = subout[1] Popt = config nth = idx elif budget_cur == max_budget: if subout[1] < Oopt: Oopt = subout[1] Popt = config nth = idx print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth) results_file.write(f"tid: {tid:d}\n") results_file.write(f" dataset = {data.I[tid][0]}\n") # results_file.write(f" Ps {data.P[tid]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close()
def main(): import matplotlib.pyplot as plt args = parse_args() ntask = args.ntask nruns = args.nruns TUNER_NAME = args.optimization Nloop = args.Nloop plot = args.plot expid = args.expid restart = args.restart (machine, processor, nodes, cores) = GetMachineConfiguration() print("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) os.environ['MACHINE_NAME'] = machine os.environ['TUNER_NAME'] = TUNER_NAME input_space = Space([Real(0., 10., transform="normalize", name="t")]) parameter_space = Space([Real(0., 1., transform="normalize", name="x")]) # input_space = Space([Real(0., 0.0001, "uniform", "normalize", name="t")]) # parameter_space = Space([Real(-1., 1., "uniform", "normalize", name="x")]) output_space = Space([Real(float('-Inf'), float('Inf'), name="y")]) constraints = {"cst1": "x >= 0. and x <= 1."} # problem = TuningProblem(input_space, parameter_space,output_space, objectives, constraints, models) # with performance model problem = TuningProblem(input_space, parameter_space, output_space, objectives, constraints, None) # no performance model computer = Computer(nodes=nodes, cores=cores, hosts=None) options = Options() options['model_restarts'] = restart options['distributed_memory_parallelism'] = False options['shared_memory_parallelism'] = False options['objective_evaluation_parallelism'] = False options['objective_multisample_threads'] = 1 options['objective_multisample_processes'] = 1 options['objective_nprocmax'] = 1 options['model_processes'] = 1 # options['model_threads'] = 1 # options['model_restart_processes'] = 1 # options['search_multitask_processes'] = 1 # options['search_multitask_threads'] = 1 # options['search_threads'] = 16 # options['mpi_comm'] = None #options['mpi_comm'] = mpi4py.MPI.COMM_WORLD options['model_class'] = 'Model_GPy_LCM' #'Model_LCM' options['verbose'] = False # options['sample_algo'] = 'MCS' # options['sample_class'] = 'SampleLHSMDU' options.validate(computer=computer) options['budget_min'] = bmin options['budget_max'] = bmax options['budget_base'] = eta smax = int( np.floor( np.log(options['budget_max'] / options['budget_min']) / np.log(options['budget_base']))) budgets = [ options['budget_max'] / options['budget_base']**x for x in range(smax + 1) ] NSs = [ int((smax + 1) / (s + 1)) * options['budget_base']**s for s in range(smax + 1) ] NSs_all = NSs.copy() budget_all = budgets.copy() for s in range(smax + 1): for n in range(s): NSs_all.append(int(NSs[s] / options['budget_base']**(n + 1))) budget_all.append(int(budgets[s] * options['budget_base']**(n + 1))) Ntotal = int(sum(NSs_all) * Nloop) Btotal = int( np.dot(np.array(NSs_all), np.array(budget_all)) / options['budget_max'] ) # total number of evaluations at highest budget -- used for single-fidelity tuners print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all) print("total number of samples: ", Ntotal) print("total number of evaluations at highest budget: ", Btotal) print(f"Sampler: {options['sample_class']}, {options['sample_algo']}") print() data = Data(problem) # giventask = [[1.0], [5.0], [10.0]] # giventask = [[1.0], [1.2], [1.3]] # giventask = [[1.0]] # t_end = args.t_end giventask = [[i] for i in np.arange(1, ntask / 2 + 1, 0.5).tolist()] # giventask = [[i] for i in np.arange(1, 1.5, 0.05).tolist()] # giventask = [[1.0], [1.05], [1.1]] NI = len(giventask) assert NI == ntask # make sure number of tasks match np.set_printoptions(suppress=False, precision=3) if (TUNER_NAME == 'GPTuneBand'): NS = Nloop data = Data(problem) gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options) (data, stats, data_hist) = gt.MB_LCM(NS=Nloop, Igiven=giventask) print("Tuner: ", TUNER_NAME) print("Sampler class: ", options['sample_class']) print("Model class: ", options['model_class']) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" t = {data.I[tid][0]:.2f}") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) nth = np.argmin(data.O[tid]) Popt = data.P[tid][nth] # find which arm and which sample the optimal param is from for arm in range(len(data_hist.P)): try: idx = (data_hist.P[arm]).index(Popt) arm_opt = arm except ValueError: pass print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx)) if (TUNER_NAME == 'GPTune'): NS = Btotal if args.nruns > 0: NS = args.nruns print("In GPTune, using the given number of nruns ", NS) NS1 = max(NS // 2, 1) gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__)) """ Building MLA with the given list of tasks """ (data, modeler, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1) print("stats: ", stats) print("model class: ", options['model_class']) print("Model restart: ", restart) """ Print all input and parameter samples """ sum_Oopt = 0. for tid in range(NI): print("tid: %d" % (tid)) print(f" t: {data.I[tid][0]:.2f} ") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], f'Oopt {min(data.O[tid])[0]:.3f}', 'nth ', np.argmin(data.O[tid])) sum_Oopt += min(data.O[tid])[0] # print("sum of all optimal objectives", sum_Oopt) if (TUNER_NAME == 'opentuner'): NS = Btotal (data, stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" t: {data.I[tid][0]:.2f} ") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS])) # single fidelity version of hpbandster if (TUNER_NAME == 'TPE'): NS = Btotal (data, stats) = HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, run_id="HpBandSter", niter=1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" t: {data.I[tid][0]:.2f} ") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) # multi-fidelity version if (TUNER_NAME == 'hpbandster'): NS = Ntotal (data, stats) = HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" t: {data.I[tid][0]:.2f} ") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) # print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) max_budget = 0. Oopt = 99999 Popt = None nth = None for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())): for subout in out[0]: budget_cur = subout[0] if budget_cur > max_budget: max_budget = budget_cur Oopt = subout[1] Popt = config nth = idx elif budget_cur == max_budget: if subout[1] < Oopt: Oopt = subout[1] Popt = config nth = idx print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth) if plot == 1: x = np.arange(0., 1., 0.0001) ymean_set = [] # stores predicted function values ytrue_set = [] for tid in range(len(data.I)): p = data.I[tid] t = p[0] fig = plt.figure(figsize=[12.8, 9.6]) I_orig = p kwargst = { input_space[k].name: I_orig[k] for k in range(len(input_space)) } y = np.zeros([len(x), 1]) y_mean = np.zeros([len(x)]) y_std = np.zeros([len(x)]) for i in range(len(x)): P_orig = [x[i]] kwargs = { parameter_space[k].name: P_orig[k] for k in range(len(parameter_space)) } kwargs.update(kwargst) y[i] = objectives(kwargs) if (TUNER_NAME == 'GPTune'): (y_mean[i], var) = predict_aug(modeler, gt, kwargs, tid) y_std[i] = np.sqrt(var) # print(y_mean[i],y_std[i],y[i]) fontsize = 40 plt.rcParams.update({'font.size': 40}) plt.plot(x, y, 'b', lw=2, label='true') plt.plot(x, y_mean, 'k', lw=3, zorder=9, label='prediction') plt.fill_between(x, y_mean - y_std, y_mean + y_std, alpha=0.2, color='k') plt.ylim(0, 2) # print(data.P[tid]) plt.scatter(data.P[tid], data.O[tid], c='r', s=50, zorder=10, edgecolors=(0, 0, 0), label='sample') plt.xlabel('x', fontsize=fontsize + 2) plt.ylabel('y(t,x)', fontsize=fontsize + 2) plt.title('t=%f' % t, fontsize=fontsize + 2) print('t:', t, 'x:', x[np.argmin(y)], 'ymin:', y.min()) # legend = plt.legend(loc='upper center', shadow=True, fontsize='x-large') legend = plt.legend(loc='upper right', shadow=False, fontsize=fontsize) annot_min(x, y) # plt.show() plt.show(block=False) plt.pause(0.5) # input("Press [enter] to continue.") # fig.savefig('obj_t_%f.eps'%t) fig.savefig(f'obj_ntask{NI}_{expid}_tid_{tid}_t_{t:.1f}.pdf') ymean_set.append(y_mean) ytrue_set.append(y) # show the distance among surrogate functions R = np.zeros( (NI, NI)) # Pearson sample correlation matrix of learned surrogates R_true = np.zeros( (NI, NI)) # Pearson sample correlation of true functions for i in range(NI): for ip in range(i, NI): ymean_i = ymean_set[i] ymean_ip = ymean_set[ip] ytrue_i = np.array((ytrue_set[i]).reshape((1, -1)))[0] ytrue_ip = np.array((ytrue_set[ip]).reshape((1, -1)))[0] # find the Pearson sample correlation coefficient R[i, ip], _ = scipy.stats.pearsonr(ymean_i, ymean_ip) R_true[i, ip], _ = scipy.stats.pearsonr(ytrue_i, ytrue_ip) print("The correlation matrix among surrogate functions is: \n", R) print("The correlation matrix among true functions is: \n", R_true) new_Rtrue = R_true[np.triu_indices(R_true.shape[0], 1)] new_R = R[np.triu_indices(R.shape[0], 1)] print("The mean absolute error is: \n", np.mean(abs(new_Rtrue - new_R))) print("The mean relative error is: \n", np.mean(abs(new_Rtrue - new_R) / abs(new_R)))
def main(): (machine, processor, nodes, cores) = GetMachineConfiguration() print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) # Parse command line arguments args = parse_args() bmin = args.bmin device = args.device bmax = args.bmax eta = args.eta nrun = args.nrun npernode = args.npernode ntask = args.ntask Nloop = args.Nloop restart = args.restart TUNER_NAME = args.optimization TLA = False (machine, processor, nodes, cores) = GetMachineConfiguration() print(args) print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) os.environ['MACHINE_NAME'] = machine os.environ['TUNER_NAME'] = TUNER_NAME # os.system("mkdir -p scalapack-driver/bin/%s; cp ../build/pdqrdriver scalapack-driver/bin/%s/.;" %(machine, machine)) ntrain = Integer(1000, 10000, transform="normalize", name="ntrain") nvalid = Integer(256, 2048, transform="normalize", name="nvalid") lr = Real(1e-6, 1e-2, name="lr") optimizer = Categoricalnorm(['Adam', 'SGD'], transform="onehot", name="optimizer") sgd_momentum = Real(0, 0.99, name="sgd_momentum") num_conv_layers = Integer(1, 3, transform="normalize", name="num_conv_layers") num_filters_1 = Integer(4, 64, transform="normalize", name="num_filters_1") num_filters_2 = Integer(4, 64, transform="normalize", name="num_filters_2") num_filters_3 = Integer(4, 64, transform="normalize", name="num_filters_3") dropout_rate = Real(0, 0.9, name="dropout_rate") num_fc_units = Integer(8, 256, transform="normalize", name="num_fc_units") validation_loss = Real(float("-Inf"), float("Inf"), name="validation_loss") IS = Space([ntrain, nvalid]) PS = Space([lr, optimizer, sgd_momentum, num_conv_layers, num_filters_1, num_filters_2, num_filters_3, dropout_rate, num_fc_units]) OS = Space([validation_loss]) constraints = {} constants={"nodes":nodes,"cores":cores,"npernode":npernode,"bmin":bmin,"bmax":bmax,"eta":eta, "device":device} print(IS, PS, OS, constraints) problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants) computer = Computer(nodes=nodes, cores=cores, hosts=None) options = Options() options['model_processes'] = 4 # parallel cholesky for each LCM kernel # options['model_threads'] = 1 # options['model_restarts'] = args.Nrestarts # options['distributed_memory_parallelism'] = False # parallel model restart options['model_restarts'] = restart options['distributed_memory_parallelism'] = False options['shared_memory_parallelism'] = False # options['mpi_comm'] = None options['model_class'] = 'Model_LCM' # Model_GPy_LCM or Model_LCM options['verbose'] = False options['budget_min'] = bmin options['budget_max'] = bmax options['budget_base'] = eta smax = int(np.floor(np.log(options['budget_max']/options['budget_min'])/np.log(options['budget_base']))) budgets = [options['budget_max'] /options['budget_base']**x for x in range(smax+1)] NSs = [int((smax+1)/(s+1))*options['budget_base']**s for s in range(smax+1)] NSs_all = NSs.copy() budget_all = budgets.copy() for s in range(smax+1): for n in range(s): NSs_all.append(int(NSs[s]/options['budget_base']**(n+1))) budget_all.append(int(budgets[s]*options['budget_base']**(n+1))) Ntotal = int(sum(NSs_all) * Nloop) Btotal = int(np.dot(np.array(NSs_all), np.array(budget_all))/options['budget_max'] * Nloop) # total number of evaluations at highest budget -- used for single-fidelity tuners print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}") print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all) print("total number of samples: ", Ntotal) print("total number of evaluations at highest budget: ", Btotal) print() options.validate(computer = computer) data = Data(problem) # giventask = [[0.2, 0.5]] if ntask == 1: giventask = [[args.ntrain, args.nvalid]] # giventask = [[3000, 1000]] NI=len(giventask) assert NI == ntask # make sure number of tasks match if(TUNER_NAME=='GPTune'): gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__)) """ Building MLA with the given list of tasks """ NS = Btotal if args.nrun > 0: NS = args.nrun NS1 = max(NS//2, 1) (data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) if(TUNER_NAME=='opentuner'): NS = Btotal (data,stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS])) # single-fidelity version of hpbandster if(TUNER_NAME=='TPE'): NS = Btotal (data,stats)=HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, run_id="HpBandSter", niter=1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) if(TUNER_NAME=='GPTuneBand'): data = Data(problem) gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options) (data, stats, data_hist)=gt.MB_LCM(NS = Nloop, Igiven = giventask) print("Tuner: ", TUNER_NAME) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) nth = np.argmin(data.O[tid]) Popt = data.P[tid][nth] # find which arm and which sample the optimal param is from for arm in range(len(data_hist.P)): try: idx = (data_hist.P[arm]).index(Popt) arm_opt = arm except ValueError: pass print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx)) # multi-fidelity version if(TUNER_NAME=='hpbandster'): NS = Ntotal (data,stats)=HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [ntrain, nvalid] = [{data.I[tid][0]}, {data.I[tid][1]}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) # print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) max_budget = 0. Oopt = 99999 Popt = None nth = None for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())): for subout in out[0]: budget_cur = subout[0] if budget_cur > max_budget: max_budget = budget_cur Oopt = subout[1] Popt = config nth = idx elif budget_cur == max_budget: if subout[1] < Oopt: Oopt = subout[1] Popt = config nth = idx print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth)
def main(): (machine, processor, nodes, cores) = GetMachineConfiguration() print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) # Parse command line arguments args = parse_args() bmin = args.bmin bmax = args.bmax eta = args.eta amin = args.amin amax = args.amax cmin = args.cmin cmax = args.cmax nprocmin_pernode = args.nprocmin_pernode ntask = args.ntask Nloop = args.Nloop restart = args.restart TUNER_NAME = args.optimization TLA = False os.environ['MACHINE_NAME'] = machine os.environ['TUNER_NAME'] = TUNER_NAME # os.system("mkdir -p scalapack-driver/bin/%s; cp ../build/pdqrdriver scalapack-driver/bin/%s/.;" %(machine, machine)) nprocmax = nodes*cores nprocmin = nodes*nprocmin_pernode a_val = Real(amin, amax, transform="normalize", name="a_val") c_val = Real(cmin, cmax, transform="normalize", name="c_val") Px = Integer(1, nprocmax, transform="normalize", name="Px") Py = Integer(1, nprocmax, transform="normalize", name="Py") Nproc = Integer(nprocmin, nprocmax, transform="normalize", name="Nproc") strong_threshold = Real(0, 1, transform="normalize", name="strong_threshold") trunc_factor = Real(0, 0.999, transform="normalize", name="trunc_factor") P_max_elmts = Integer(1, 12, transform="normalize", name="P_max_elmts") # coarsen_type = Categoricalnorm (['0', '1', '2', '3', '4', '6', '8', '10'], transform="onehot", name="coarsen_type") coarsen_type = Categoricalnorm (['0', '1', '2', '3', '4', '8', '10'], transform="onehot", name="coarsen_type") relax_type = Categoricalnorm (['-1', '0', '6', '8', '16', '18'], transform="onehot", name="relax_type") smooth_type = Categoricalnorm (['5', '6', '8', '9'], transform="onehot", name="smooth_type") smooth_num_levels = Integer(0, 5, transform="normalize", name="smooth_num_levels") interp_type = Categoricalnorm (['0', '3', '4', '5', '6', '8', '12'], transform="onehot", name="interp_type") agg_num_levels = Integer(0, 5, transform="normalize", name="agg_num_levels") r = Real(float("-Inf"), float("Inf"), name="r") IS = Space([a_val, c_val]) PS = Space([Px, Py, Nproc, strong_threshold, trunc_factor, P_max_elmts, coarsen_type, relax_type, smooth_type, smooth_num_levels, interp_type, agg_num_levels]) OS = Space([r]) cst1 = f"Px * Py <= Nproc" cst2 = f"not(P_max_elmts==10 and coarsen_type=='6' and relax_type=='18' and smooth_type=='6' and smooth_num_levels==3 and interp_type=='8' and agg_num_levels==1)" constraints = {"cst1": cst1,"cst2": cst2} constants={"nodes":nodes,"cores":cores,"bmin":bmin,"bmax":bmax,"eta":eta} print(IS, PS, OS, constraints) problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants) computer = Computer(nodes=nodes, cores=cores, hosts=None) options = Options() options['model_processes'] = 4 # parallel cholesky for each LCM kernel # options['model_threads'] = 1 # options['model_restarts'] = args.Nrestarts # options['distributed_memory_parallelism'] = False # parallel model restart options['model_restarts'] = restart options['distributed_memory_parallelism'] = False options['shared_memory_parallelism'] = False # options['mpi_comm'] = None options['model_class'] = 'Model_LCM' # Model_GPy_LCM or Model_LCM options['verbose'] = False # choose sampler # options['sample_class'] = 'SampleOpenTURNS' if args.lhs == 1: options['sample_class'] = 'SampleLHSMDU' options['sample_algo'] = 'LHS-MDU' options.validate(computer=computer) options['budget_min'] = bmin options['budget_max'] = bmax options['budget_base'] = eta smax = int(np.floor(np.log(options['budget_max']/options['budget_min'])/np.log(options['budget_base']))) budgets = [options['budget_max'] /options['budget_base']**x for x in range(smax+1)] NSs = [int((smax+1)/(s+1))*options['budget_base']**s for s in range(smax+1)] NSs_all = NSs.copy() budget_all = budgets.copy() for s in range(smax+1): for n in range(s): NSs_all.append(int(NSs[s]/options['budget_base']**(n+1))) budget_all.append(int(budgets[s]*options['budget_base']**(n+1))) Ntotal = int(sum(NSs_all) * Nloop) Btotal = int(np.dot(np.array(NSs_all), np.array(budget_all))/options['budget_max'] * Nloop) # total number of evaluations at highest budget -- used for single-fidelity tuners print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}") print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all) print("total number of samples: ", Ntotal) print("total number of evaluations at highest budget: ", Btotal) print() data = Data(problem) giventask = [[(amax-amin)*random()+amin,(cmax-cmin)*random()+cmin] for i in range(ntask)] # giventask = [[0.2, 0.5]] if ntask == 1: giventask = [[args.a, args.c]] NI=len(giventask) assert NI == ntask # make sure number of tasks match # # the following will use only task lists stored in the pickle file # data = Data(problem) if(TUNER_NAME=='GPTune'): gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__)) """ Building MLA with the given list of tasks """ NS = Btotal if args.nrun > 0: NS = args.nrun NS1 = max(NS//2, 1) (data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) if TLA is True: """ Call TLA for 2 new tasks using the constructed LCM model""" newtask = [[0.5, 0.3], [0.2, 1.0]] (aprxopts, objval, stats) = gt.TLA1(newtask, NS=None) print("stats: ", stats) """ Print the optimal parameters and function evaluations""" for tid in range(len(newtask)): print("new task: %s" % (newtask[tid])) print(' predicted Popt: ', aprxopts[tid], ' objval: ', objval[tid]) if(TUNER_NAME=='opentuner'): NS = Btotal (data,stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS])) # single-fidelity version of hpbandster if(TUNER_NAME=='TPE'): NS = Btotal (data,stats)=HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, run_id="HpBandSter", niter=1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) if(TUNER_NAME=='GPTuneBand'): data = Data(problem) gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options) (data, stats, data_hist)=gt.MB_LCM(NS = Nloop, Igiven = giventask) print("Tuner: ", TUNER_NAME) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) nth = np.argmin(data.O[tid]) Popt = data.P[tid][nth] # find which arm and which sample the optimal param is from for arm in range(len(data_hist.P)): try: idx = (data_hist.P[arm]).index(Popt) arm_opt = arm except ValueError: pass print(' Popt ', Popt, 'Oopt ', min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx)) if(TUNER_NAME=='GPTuneBand_single'): def merge_dict(mydict, newdict): for key in mydict.keys(): mydict[key] += newdict[key] data_all = [] stats_all = {} for singletask in giventask: NI = 1 cur_task = [singletask] data = Data(problem) gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options) (data, stats)=gt.MB_LCM(NS = Nloop, Igiven = cur_task) data_all.append(data) merge_dict(stats_all, stats) print("Finish one single task tuning") print("Tuner: ", TUNER_NAME) print("stats: ", stats) tid = 0 print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) print("Finish tuning...") print("Tuner: ", TUNER_NAME) print("stats_all: ", stats_all) for i in range(len(data_all)): data = data_all[i] for tid in range(NI): print("tid: %d" % (i)) print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) # multi-fidelity version if(TUNER_NAME=='hpbandster'): NS = Ntotal (data,stats)=HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(f" [a_val, c_val] = [{data.I[tid][0]:.3f}, {data.I[tid][1]:.3f}]") print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) # print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) max_budget = 0. Oopt = 99999 Popt = None nth = None for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())): for subout in out[0]: budget_cur = subout[0] if budget_cur > max_budget: max_budget = budget_cur Oopt = subout[1] Popt = config nth = idx elif budget_cur == max_budget: if subout[1] < Oopt: Oopt = subout[1] Popt = config nth = idx print(' Popt ', Popt, 'Oopt ', Oopt, 'nth ', nth)
def main(): # Parse command line arguments args = parse_args() # Extract arguments ntask = args.ntask npernode = args.npernode optimization = args.optimization nrun = args.nrun bmin = args.bmin bmax = args.bmax eta = args.eta Nloop = args.Nloop restart = args.restart expid = args.expid TUNER_NAME = args.optimization (machine, processor, nodes, cores) = GetMachineConfiguration() ot.RandomGenerator.SetSeed(args.seed) print(args) print ("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) os.environ['MACHINE_NAME'] = machine os.environ['TUNER_NAME'] = TUNER_NAME datasets = args.dataset.split('-') datafiles = [] giventask = [] assert len(datasets) == ntask for i in range(len(datasets)): datafiles.append(f"data/{datasets[i]}") giventask.append([f"data/{datasets[i]}"]) # datafiles = ["data/branin"] # Task input parameters datafile = Categoricalnorm(datafiles, transform="onehot", name="datafile") # Tuning parameters h = Real(-10, 10, transform="normalize", name="h") Lambda = Real(-10, 10, transform="normalize", name="Lambda") # npernode = Integer(int(math.log2(nprocmin_pernode)), int(math.log2(cores)), transform="normalize", name="npernode") result = Real(0 , float("Inf"),name="r") IS = Space([datafile]) # PS = Space([h,Lambda,npernode]) PS = Space([h,Lambda]) OS = Space([result]) constraints = {} models = {} constants={"nodes":nodes,"cores":cores,"npernode":npernode,"bmin":bmin,"bmax":bmax,"eta":eta} """ Print all input and parameter samples """ print(IS, PS, OS, constraints, models) problem = TuningProblem(IS, PS, OS, objectives, constraints, constants=constants) computer = Computer(nodes = nodes, cores = cores, hosts = None) """ Set and validate options """ options = Options() options['model_processes'] = 1 # options['model_threads'] = 1 options['model_restarts'] = 1 # options['search_multitask_processes'] = 1 # options['model_restart_processes'] = 1 options['distributed_memory_parallelism'] = False options['shared_memory_parallelism'] = False options['model_class '] = 'Model_LCM' # 'Model_GPy_LCM' options['verbose'] = False options['sample_class'] = 'SampleOpenTURNS' options['budget_min'] = bmin options['budget_max'] = bmax options['budget_base'] = eta smax = int(np.floor(np.log(options['budget_max']/options['budget_min'])/np.log(options['budget_base']))) budgets = [options['budget_max'] /options['budget_base']**x for x in range(smax+1)] NSs = [int((smax+1)/(s+1))*options['budget_base']**s for s in range(smax+1)] NSs_all = NSs.copy() budget_all = budgets.copy() for s in range(smax+1): for n in range(s): NSs_all.append(int(NSs[s]/options['budget_base']**(n+1))) budget_all.append(int(budgets[s]*options['budget_base']**(n+1))) Ntotal = int(sum(NSs_all) * Nloop) Btotal = int(np.dot(np.array(NSs_all), np.array(budget_all))/options['budget_max'] * Nloop) # total number of evaluations at highest budget -- used for single-fidelity tuners print(f"bmin = {bmin}, bmax = {bmax}, eta = {eta}, smax = {smax}") print("samples in one multi-armed bandit loop, NSs_all = ", NSs_all) print("total number of samples: ", Ntotal) print("total number of evaluations at highest budget: ", Btotal) print() options.validate(computer = computer) # """ Building MLA with the given list of tasks """ data = Data(problem) NI = ntask if(TUNER_NAME=='GPTune'): gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__)) NS = Btotal if args.nrun > 0: NS = args.nrun NS1 = max(NS//2, 1) (data, model, stats) = gt.MLA(NS=NS, NI=NI, Igiven=giventask, NS1=NS1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d"%(tid)) print(" matrix:%s"%(data.I[tid][0])) print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', -min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) results_file.write(f"tid: {tid:d}\n") results_file.write(f" matrix:{data.I[tid][0]:s}\n") # results_file.write(f" Ps {data.P[tid]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() if(TUNER_NAME=='opentuner'): NS = Btotal if args.nrun > 0: NS = args.nrun NS1 = max(NS//2, 1) (data,stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d"%(tid)) print(" matrix:%s"%(data.I[tid][0])) print(" Ps ", data.P[tid][:NS]) print(" Os ", data.O[tid][:NS]) print(' Popt ', data.P[tid][np.argmin(data.O[tid][:NS])], 'Oopt ', -min(data.O[tid][:NS])[0], 'nth ', np.argmin(data.O[tid][:NS])) results_file.write(f"tid: {tid:d}\n") results_file.write(f" matrix:{data.I[tid][0]:s}\n") # results_file.write(f" Ps {data.P[tid][:NS]}\n") results_file.write(f" Os {data.O[tid][:NS].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() if(TUNER_NAME=='TPE'): NS = Btotal if args.nrun > 0: NS = args.nrun NS1 = max(NS//2, 1) (data,stats)=HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="HpBandSter", niter=1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d"%(tid)) print(" matrix:%s"%(data.I[tid][0])) print(" Ps ", data.P[tid]) print(" Os ", data.O[tid]) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', -min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) results_file.write(f"tid: {tid:d}\n") results_file.write(f" matrix:{data.I[tid][0]:s}\n") # results_file.write(f" Ps {data.P[tid]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() if(TUNER_NAME=='GPTuneBand'): data = Data(problem) gt = GPTune_MB(problem, computer=computer, NS=Nloop, options=options) (data, stats, data_hist)=gt.MB_LCM(NS = Nloop, Igiven = giventask) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(" matrix:%s"%(data.I[tid][0])) print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) nth = np.argmin(data.O[tid]) Popt = data.P[tid][nth] # find which arm and which sample the optimal param is from for arm in range(len(data_hist.P)): try: idx = (data_hist.P[arm]).index(Popt) arm_opt = arm except ValueError: pass print(' Popt ', Popt, 'Oopt ', -min(data.O[tid])[0], 'nth ', nth, 'nth-bandit (s, nth) = ', (arm_opt, idx)) results_file.write(f"tid: {tid:d}\n") results_file.write(f" matrix:{data.I[tid][0]:s}\n") # results_file.write(f" Ps {data.P[tid]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close() # multi-fidelity version if(TUNER_NAME=='hpbandster'): NS = Ntotal (data,stats)=HpBandSter_bandit(T=giventask, NS=NS, tp=problem, computer=computer, options=options, run_id="hpbandster_bandit", niter=1) print("Tuner: ", TUNER_NAME) print("stats: ", stats) results_file = open(f"KRR_{args.dataset}_ntask{args.ntask}_bandit{args.bmin}-{args.bmax}-{args.eta}_Nloop{args.Nloop}_expid{args.expid}.txt", "a") results_file.write(f"Tuner: {TUNER_NAME}\n") results_file.write(f"stats: {stats}\n") """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(" matrix:%s"%(data.I[tid][0])) # print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) # print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) max_budget = 0. Oopt = 99999 Popt = None nth = None for idx, (config, out) in enumerate(zip(data.P[tid], data.O[tid].tolist())): for subout in out[0]: budget_cur = subout[0] if budget_cur > max_budget: max_budget = budget_cur Oopt = subout[1] Popt = config nth = idx elif budget_cur == max_budget: if subout[1] < Oopt: Oopt = subout[1] Popt = config nth = idx print(' Popt ', Popt, 'Oopt ', -Oopt, 'nth ', nth) results_file.write(f"tid: {tid:d}\n") results_file.write(f" matrix:{data.I[tid][0]:s}\n") # results_file.write(f" Ps {data.P[tid]}\n") results_file.write(f" Os {data.O[tid].tolist()}\n") # results_file.write(f' Popt {data.P[tid][np.argmin(data.O[tid])]} Oopt {-min(data.O[tid])[0]} nth {np.argmin(data.O[tid])}\n') results_file.close()