def __init__(self, task='yacht', n_units=50, activation='tanh', seed=0): # ------------ saving paths created ------------ self.saving_model_path = None # ------------ generate train and test data ------------ np.random.seed(seed) # ------------ generate data ------------ if task in ['yacht', 'concrete', 'wine-quality-red', 'bostonHousing']: X_train, y_train, X_test, y_test = load_uci_data( data_directory=f'exps_tasks/datasets/{task}', split_number=seed) else: maths_f, x_bounds, _, true_fmin = get_function(task) n_train = 5000 n_test = 2000 X_train, y_train = get_init_data(obj_func=maths_f, noise_var=1e-6, n_init=n_train, bounds=x_bounds) X_test, y_test = get_init_data(obj_func=maths_f, noise_var=1e-6, n_init=n_test, bounds=x_bounds) # X_train, y_train, X_test, y_test = load_uci_data(data_directory=f'./datasets/{task}', split_number=seed) self.X_train = X_train self.y_train = y_train.flatten() self.X_test = X_test self.y_test = y_test.flatten() # ------------ bnn hyperparameters ------------ self.num_samples = 300 self.keep_every = 3 self.activation = activation self.seed = seed # bnds for hyperparameters to be tuned self.true_bnds = np.array([ [2, 50.0], # num_steps_per_sample [100, 5000], # tau_out [1e-3, 1e-1], # length_scale [1e-4, 5e-2] ] # step_size ) self.d = len(self.true_bnds)
def test_gp(): # Specify the objective function and parameters (noise variance, input dimension, initial observation np.random.seed(3) x_ob, y_ob = get_init_data(obj_func=f, noise_var=var_noise, n_init=n_init, bounds=x_bounds) # ------ Test grid -------- # if d == 2: x1, x2 = np.mgrid[-1:1:50j, -1:1:50j] X = np.vstack((x1.flatten(), x2.flatten())).T y = f(X) else: X = np.linspace(-1, 1, 100)[:, None] y = f(X) # -- GP model ----# # kern = GPy.kern.Matern52(2, variance=1., ARD=True) GP = GPModel(exact_feval=True, ARD=True) GP._update_model(x_ob, y_ob) m, s = GP.predict(X) # # ------ Plot figures -------- # if d == 2: figure, axes = plt.subplots(2, 1, figsize=(10, 10)) sub1 = axes[0].contourf(x1, x2, y.reshape(50, 50)) axes[0].plot(x_ob[:, 0], x_ob[:, 1], 'rx') axes[0].set_title('objective func ') sub2 = axes[1].contourf(x1, x2, m.reshape(50, 50)) axes[1].plot(x_ob[:, 0], x_ob[:, 1], 'rx') pred_title = f'prediction by GP' axes[1].set_title(pred_title) plt.show() else: figure, axes = plt.subplots(1, 1, figsize=(10, 10)) axes.plot(x_ob, y_ob, 'rx') axes.plot(X, y, 'r--') axes.plot(X, m, 'b') axes.fill_between(X.flatten(), (m - s).flatten(), (m + s).flatten(), color='blue', alpha=0.30) axes.set_title('1D Regression') plt.show() # ------ Save figures -------- # saving_path = 'data/syntheticFns/' if not os.path.exists(saving_path): os.makedirs(saving_path) fig_name = func_name + f'_gp.png' print(fig_name)
def test_compare_gp_with_dropnet(): # Specify the objective function and parameters (noise variance, input dimension, initial observation np.random.seed(3) x_ob, y_ob = get_init_data(obj_func=f, noise_var=var_noise, n_init =n_init, bounds=x_bounds) # ------ Test grid -------- # if d == 2: x1, x2 = np.mgrid[-1:1:50j, -1:1:50j] X = np.vstack((x1.flatten(), x2.flatten())).T y = f(X) # Dropout NN Configuration dropout = 0.05 T = 100 tau = 1.0 bs = 10 tbs = 50 n_hidden = [50, 50, 50] else: X = np.linspace(-1,1,100)[:,None] y = f(X) # Dropout NN Configuration dropout = 0.05 T = 100 tau = 20 bs = 10 tbs = 50 n_hidden = [50, 50, 50] # n_hidden = [100] # -- GP model ----# # kern = GPy.kern.Matern52(2, variance=1., ARD=True) GP = GPModel(exact_feval=True,ARD=True) GP._update_model(x_ob, y_ob) m_gp, s_gp = GP.predict(X) # -- DropoutNet model ----# DropNet = DropoutNet(n_epochs=40, n_hidden=n_hidden, dropout=dropout, T=T, tau=tau, batch_size=bs) DropNet._update_model(x_ob, y_ob) m_drop, s_drop = DropNet.predict(X, test_batch_size=tbs) # # ------ Plot figures -------- # if d == 2: figure, axes = plt.subplots(3, 1, figsize=(6, 18)) sub1 = axes[0].contourf(x1, x2, y.reshape(50, 50)) axes[0].plot(x_ob[:,0], x_ob[:,1],'rx') axes[0].set_title('objective func ') sub2 = axes[1].contourf(x1, x2, m_gp.reshape(50, 50)) axes[1].plot(x_ob[:,0], x_ob[:,1],'rx') gp_title=f'prediction by GP' axes[1].set_title(gp_title) sub2 = axes[2].contourf(x1, x2, m_drop.reshape(50, 50)) axes[2].plot(x_ob[:,0], x_ob[:,1],'rx') dropnet_title=f'prediction by NN:dropout={dropout},T={T},tau={tau},BS={bs},TBS={tbs}' axes[2].set_title(dropnet_title) plt.show() else: figure, axes = plt.subplots(2, 1, figsize=(10, 8)) axes[0].plot(x_ob,y_ob, 'rx') axes[0].plot(X, y, 'r--') axes[0].plot(X, m_gp, 'b') axes[0].fill_between(X.flatten(), (m_gp - s_gp).flatten(), (m_gp + s_gp).flatten(), color='blue', alpha=0.30) axes[0].set_title('1D GP Regression') axes[1].plot(x_ob,y_ob, 'rx') axes[1].plot(X, y, 'r--') axes[1].plot(X, m_drop, 'b') axes[1].fill_between(X.flatten(), (m_drop - s_drop).flatten(), (m_drop + s_drop).flatten(), color='blue', alpha=0.30) dropnet_title=f'prediction by NN:dropout={dropout},T={T},tau={tau},BS={bs},TBS={tbs}' axes[1].set_title(dropnet_title) plt.show() # ------ Save figures -------- # saving_path = 'data/syntheticFns/' if not os.path.exists(saving_path): os.makedirs(saving_path) # gp_fig_name = func_name + f'_gp.png' # dropnet_fig_name = func_name + f'_gp.png' fig_name = 'compare_gp_dropnet.png'
def BayesOpt_attack(obj_func, model_type, acq_type, batch_size, low_dim, sparse, seed, img_offset, n_init=50, num_iter=40, ntargets=9, target_label=0, dim_reduction='BILI', cost_metric=None, obj_metric=1, update_freq=10, nsubspaces=1): # Specify code directory directory = './' if obj_func == 'mnist': high_dim = 784 nchannel = 1 epsilon = 0.3 elif obj_func == 'cifar10': high_dim = int(32 * 32) nchannel = 3 epsilon = 0.05 elif obj_func == 'imagenet': high_dim = int(96 * 96) nchannel = 3 epsilon = 0.05 ntargets = 1 if 'LDR' in model_type: low_dim = high_dim if dim_reduction == 'NONE': x_bounds = np.vstack([[-1, 1]] * high_dim * nchannel) else: x_bounds = np.vstack([[-1, 1]] * low_dim * nchannel) # Specify the experiment results saving directory results_data_folder = f'{directory}exp_results/{obj_func}_tf_{model_type}_ob{obj_metric}_' \ f'_freq{update_freq}_ld{low_dim}_{dim_reduction}/' if not os.path.exists(results_data_folder): os.makedirs(results_data_folder) # Define the model and the original images to be attacked cnn = CNN(dataset_name=obj_func, img_offset=img_offset, epsilon=epsilon, dim_reduction=dim_reduction, low_dim=low_dim, high_dim=high_dim, obj_metric=obj_metric, results_folder=results_data_folder, directory=directory) # For each image, define the target class if ntargets > 1: target_list = list(range(ntargets)) else: target_list = [target_label] # Start attack each target in sequence for tg in target_list: cnn.get_data_sample(tg) input_label = cnn.input_label img_id = cnn.orig_img_id target_label = cnn.target_label[0] print( f'id={img_offset}, origin={input_label}, target={target_label}, eps={epsilon}, dr={low_dim}' ) # Define the BO objective function if obj_func == 'imagenet': if 'LDR' in model_type or dim_reduction == 'NONE': f = lambda x: cnn.np_evaluate_bili(x) else: f = lambda x: cnn.np_upsample_evaluate_bili(x) else: if 'LDR' in model_type or dim_reduction == 'NONE': f = lambda x: cnn.np_evaluate(x) else: f = lambda x: cnn.np_upsample_evaluate(x) # Define the name of results file and failure fail(for debug or resume) results_file_name = os.path.join( results_data_folder, f'{model_type}{acq_type}{batch_size}_{dim_reduction}_' f'd{low_dim}_i{input_label}_t{target_label}_id{img_id}') failed_file_name = os.path.join( results_data_folder, f'failed_{model_type}{acq_type}{batch_size}_{dim_reduction}_' f'd{low_dim}_i{input_label}_t{target_label}_id{img_id}') X_opt_all_slices = [] Y_opt_all_slices = [] X_query_all_slices = [] Y_query_all_slices = [] X_reduced_opt_all_slices = [] X_reduced_query_all_slices = [] seed_list = [seed] # can be modified to do BO over multiple seeds for seed in seed_list: # Specify the random seed np.random.seed(seed) # Generate initial observation data for BO if os.path.exists(results_file_name) and 'LDR' not in model_type: print('load old init data') with open(results_file_name, 'rb') as pre_file: previous_bo_results = pickle.load(pre_file) x_init = previous_bo_results['X_reduced_query'][0] y_init = previous_bo_results['Y_query'][0] else: print('generate new init data') x_init, y_init = get_init_data(obj_func=f, n_init=n_init, bounds=x_bounds) print(f'X init shape {x_init.shape}') # Initialise BO bayes_opt = Bayes_opt(func=f, bounds=x_bounds, saving_path=failed_file_name) bayes_opt.initialise(X_init=x_init, Y_init=y_init, model_type=model_type, acq_type=acq_type, sparse=sparse, nsubspaces=nsubspaces, batch_size=batch_size, update_freq=update_freq, nchannel=nchannel, high_dim=high_dim, dim_reduction=dim_reduction, cost_metric=cost_metric, seed=seed) # Run BO X_query_full, Y_query, X_opt_full, Y_opt, time_record = bayes_opt.run( total_iterations=num_iter) # Reduce the memory needed for storing results if 'LDR' in model_type: X_query = X_query_full[-2:] X_opt = X_opt_full[-2:] else: X_query = X_query_full X_opt = X_opt_full[-2:] # Store the results Y_opt_all_slices.append(Y_opt) Y_query_all_slices.append(Y_query) opt_dr_list = bayes_opt.opt_dr_list if dim_reduction == 'NONE': X_reduced_opt_all_slices.append(X_opt.astype(np.float16)) X_reduced_query_all_slices.append(X_query.astype(np.float16)) X_query_all_slices.append(X_query) X_opt_all_slices.append(X_opt) print( f'Y_opt={Y_opt[-1]}, X_opt shape{X_opt.shape}, X_h_opt shape{X_opt.shape}, ' f'X_query shape{X_query.shape}, X_h_query shape{X_query.shape}, opt_dr={opt_dr_list[-1]}' ) else: X_reduced_opt_all_slices.append(X_opt.astype(np.float16)) X_reduced_query_all_slices.append(X_query.astype(np.float16)) # Transform data from reduced search space to original high-dimensional input space X_h_query = upsample_projection(dim_reduction, X_query, low_dim=low_dim, high_dim=high_dim, nchannel=nchannel) X_query_all_slices.append(X_h_query[-2:]) X_h_opt = upsample_projection(dim_reduction, X_opt, low_dim=low_dim, high_dim=high_dim, nchannel=nchannel) X_opt_all_slices.append(X_h_opt) print( f'Y_opt={Y_opt[-1]}, X_opt shape{X_opt.shape}, X_h_opt shape{X_h_opt.shape}, ' f'X_query shape{X_query.shape}, X_h_query shape{X_h_query.shape}' ) # For ImageNet images, save only the L_inf norm and L2 norm instead of the adversarial image if 'imagenet' in obj_func: l_inf_sum = np.abs(X_h_opt[-1, :]).sum() l_2_norm = np.sqrt( np.sum((epsilon * X_h_opt[-1, :].ravel())**2)) X_opt_all_slices = [l_inf_sum] X_query_all_slices = [l_2_norm] # Save the results locally results = { 'X_opt': X_opt_all_slices, 'Y_opt': Y_opt_all_slices, 'X_query': X_query_all_slices, 'Y_query': Y_query_all_slices, 'X_reduced_opt': X_reduced_opt_all_slices, 'X_reduced_query': X_reduced_query_all_slices, 'dr_opt_list': opt_dr_list, 'runtime': time_record } with open(results_file_name, 'wb') as file: pickle.dump(results, file)
def test_compare_gp_with_dropnet(): # Specify the objective function and parameters (noise variance, input dimension, initial observation np.random.seed(3) x_ob, y_ob = get_init_data(obj_func=f, noise_var=var_noise, n_init=n_init, bounds=x_bounds) rng = np.random.RandomState(42) x = rng.rand(n_init) # ------ Test grid -------- # if d == 2: x1, x2 = np.mgrid[-1:1:50j, -1:1:50j] X = np.vstack((x1.flatten(), x2.flatten())).T y = f(X) # Dropout NN Configuration dropout = 0.05 T = 100 tau = 1.0 bs = 10 tbs = 50 n_hidden = [50, 50, 50] else: X = np.linspace(-1, 1, 100)[:, None] y = f(X) # Dropout NN Configuration dropout = 0.05 T = 100 tau = 20 bs = 10 tbs = 50 n_hidden = [50, 50, 50] # n_hidden = [100] # -- GP model ----# GP = GPModel(exact_feval=True, ARD=True) GP._update_model(x_ob, y_ob) m_gp, s_gp = GP.predict(X) # -- MCDropoutNet or DNGO model ----# Net = DNGOWrap() # Net = MCDROPWarp() # Net = BOHAMIANNWarp(num_samples=600) Net._update_model(x_ob, y_ob) m_net, s_net = Net.predict(X) # # ------ Plot figures -------- # if d == 2: figure, axes = plt.subplots(3, 1, figsize=(6, 18)) sub1 = axes[0].contourf(x1, x2, y.reshape(50, 50)) axes[0].plot(x_ob[:, 0], x_ob[:, 1], 'rx') axes[0].set_title('objective func ') sub2 = axes[1].contourf(x1, x2, m_gp.reshape(50, 50)) axes[1].plot(x_ob[:, 0], x_ob[:, 1], 'rx') gp_title = f'prediction by GP' axes[1].set_title(gp_title) sub2 = axes[2].contourf(x1, x2, m_net.reshape(50, 50)) axes[2].plot(x_ob[:, 0], x_ob[:, 1], 'rx') dropnet_title = f'prediction by NN:dropout={dropout},T={T},tau={tau},BS={bs},TBS={tbs}' axes[2].set_title(dropnet_title) plt.show() else: figure, axes = plt.subplots(2, 1, figsize=(10, 8)) axes[0].plot(x_ob, y_ob, 'ro') axes[0].plot(X, y, 'k--') axes[0].plot(X, m_gp, 'b') axes[0].fill_between(X.flatten(), (m_gp - s_gp).flatten(), (m_gp + s_gp).flatten(), color='orange', alpha=0.80) axes[0].fill_between(X.flatten(), (m_gp - 2 * s_gp).flatten(), (m_gp + 2 * s_gp).flatten(), color='orange', alpha=0.60) axes[0].fill_between(X.flatten(), (m_gp - 3 * s_gp).flatten(), (m_gp + 3 * s_gp).flatten(), color='orange', alpha=0.40) axes[0].set_title('1D GP Regression') axes[1].plot(x_ob, y_ob, 'ro') axes[1].plot(X, y, 'k--') axes[1].plot(X, m_net, 'b') axes[1].fill_between(X.flatten(), (m_net - s_net).flatten(), (m_net + s_net).flatten(), color='orange', alpha=0.80) axes[1].fill_between(X.flatten(), (m_net - 2 * s_net).flatten(), (m_net + 2 * s_net).flatten(), color='orange', alpha=0.60) axes[1].fill_between(X.flatten(), (m_net - 3 * s_net).flatten(), (m_net + 3 * s_net).flatten(), color='orange', alpha=0.40) dropnet_title = f'prediction by NN:dropout={dropout},T={T},tau={tau},BS={bs},TBS={tbs}' axes[1].set_title(dropnet_title) plt.show() # ------ Save figures -------- # saving_path = 'data/syntheticFns/' if not os.path.exists(saving_path): os.makedirs(saving_path) # gp_fig_name = func_name + f'_gp.png' # dropnet_fig_name = func_name + f'_gp.png' fig_name = 'compare_gp_dropnet.png'
def BNN_BO_Exps(obj_func, model_type, bo_method, batch_option, batch_size, num_iter=40, seed_size=20, util_type='se_y', activation='tanh'): # Specify the objective function and parameters (noise variance, input dimension, number of initial observation) f, x_bounds, _, true_fmin = get_function(obj_func) var_noise = 1.0e-10 d = x_bounds.shape[0] n_init = d * 10 saving_path = 'data/' + obj_func if not os.path.exists(saving_path): os.makedirs(saving_path) if model_type == 'LCBNN' or model_type == 'LCCD': results_file_name = saving_path + '/' + model_type + activation + util_type + bo_method + str( batch_size) else: results_file_name = saving_path + '/' + model_type + activation + bo_method + str( batch_size) if os.path.exists(results_file_name): with open(results_file_name, 'rb') as exist_data_filefile2: existing_results = pickle.load(exist_data_filefile2) X_opt_all_seeds = existing_results['X_opt'] Y_opt_all_seeds = existing_results['Y_opt'] X_query_all_seeds = existing_results['X_query'] Y_query_all_seeds = existing_results['Y_query'] time_all_seeds = existing_results['runtime'] if isinstance(X_query_all_seeds, list): X_query_all_seeds = X_query_all_seeds Y_query_all_seeds = Y_query_all_seeds time_all_seeds = time_all_seeds else: X_query_all_seeds = list(X_query_all_seeds) Y_query_all_seeds = list(Y_query_all_seeds) time_all_seeds = list(time_all_seeds) s_start = len(Y_opt_all_seeds) print(f"Using existing data from seed{s_start} onwards") else: s_start = 0 X_opt_all_seeds = [] Y_opt_all_seeds = [] X_query_all_seeds = [] Y_query_all_seeds = [] time_all_seeds = [] for j in range(s_start, seed_size): # specify the random seed and generate observation data seed = j np.random.seed(seed) x_init, y_init = get_init_data(obj_func=f, noise_var=var_noise, n_init=n_init, bounds=x_bounds) # run Bayesian optimisation: bayes_opt = Bayes_opt(func=f, bounds=x_bounds, noise_var=var_noise) # model_type: GP or MCDROP or DNGO or BOHAM bayes_opt.initialise(X_init=x_init, Y_init=y_init, model_type=model_type, bo_method=bo_method, batch_option=batch_option, batch_size=batch_size, seed=seed, util_type=util_type, actv_func=activation) # output of Bayesian optimisation: X_query, Y_query, X_opt, Y_opt, time_record = bayes_opt.iteration_step( iterations=num_iter) # X_query, Y_query - query points selected by BO; # X_opt, Yopt - guesses of the global optimum/optimiser (= optimum point of GP posterior mean) # store data X_opt_all_seeds.append(X_opt) Y_opt_all_seeds.append(Y_opt) X_query_all_seeds.append(X_query) Y_query_all_seeds.append(Y_query) time_all_seeds.append(time_record) results = { 'X_opt': X_opt_all_seeds, 'Y_opt': Y_opt_all_seeds, 'X_query': X_query_all_seeds, 'Y_query': Y_query_all_seeds, 'runtime': time_all_seeds } with open(results_file_name, 'wb') as file: pickle.dump(results, file)