def imdb_run(): args = parser.parse_args() data = args.data freeze = args.freeze nlayer = args.nlayer kept_prob = args.kept_prob_dropout bert_lstm_save_path = args.save_path learning_rate = args.learning_rate epoches = args.epoches tokenizer_selection = args.tokenizer if data.lower() == 'imdb': data_path = 'aclImdb' bert = BertModel.from_pretrained('bert-base-uncased') tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') max_len = 100 # max_vocab = bert.config.to_dict()['vocab_size']-3 # data_processed = pre_processing(data_path, max_vocab) # train_sequences, test_sequences = data_processed.seqs_num() # train_text_init, test_text_init = data_processed.numerical(train_sequences, test_sequences, max_len = max_len) max_vocab = 50000 data_processed = pre_processing(data_path, max_vocab, max_len) if tokenizer_selection.lower() != 'bert': data_processed.processing() train_sequences, test_sequences = data_processed.bert_indx(tokenizer) print('Self preprocessing') else: data_processed.bert_tokenize(tokenizer) train_sequences, test_sequences = data_processed.bert_indx(tokenizer) print('BERT tokenizer') train_text_init, test_text_init = data_processed.numerical( tokenizer, train_sequences, test_sequences) train_text = pad_sequences(train_text_init, maxlen=max_len, padding='post') test_text = pad_sequences(test_text_init, maxlen=max_len, padding='post') train_target = data_processed.all_train_labels test_target = data_processed.all_test_labels all_train_data, train_data, vali_data, test_data = data_loading( train_text, test_text, train_target, test_target) device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') print(device) BatchSize = 128 #int(length_train/200) all_train_loader = DataLoader(all_train_data, batch_size=BatchSize, shuffle=True) train_loader = DataLoader(train_data, batch_size=BatchSize, shuffle=True) vali_loader = DataLoader(vali_data, batch_size=BatchSize, shuffle=True) test_loader = DataLoader(test_data, batch_size=BatchSize, shuffle=True) bidirection = args.bidirection model = bert_lstm(bert, 2, bidirection, nlayer, 128, freeze, kept_prob) model.to(device) criterion = nn.CrossEntropyLoss() optimiser = torch.optim.AdamW( [cont for cont in model.parameters() if cont.requires_grad], lr=learning_rate) bert_lstm_save_path = os.path.join( bert_lstm_save_path, 'best_bert_' + str(kept_prob) + '_' + str(learning_rate) + '_' + str(tokenizer_selection) + '_' + str(max_len)) best_epoch = 0 best_acc = 0 patience = 20 for epoch in range(epoches): test_pred = torch.tensor([]) test_targets = torch.tensor([]) train_pred = torch.tensor([]) train_targets = torch.tensor([]) test_loss = [] train_loss = [] model.train() for batch_index, (seqs, length, target) in enumerate(all_train_loader): seqs = seqs.type(torch.LongTensor) args = torch.argsort(length, descending=True) length = length[args] seqs = seqs[args][:, 0:length[0]] target = target[args].type(torch.LongTensor) optimiser.zero_grad() seqs, target, length = seqs.to(device), target.to( device), length.to(device) output, pred_out = model(seqs, length, True) loss = criterion(output, target) loss.backward() optimiser.step() train_pred = torch.cat((train_pred, pred_out.cpu()), dim=0) train_targets = torch.cat( (train_targets, target.type(torch.float).cpu())) train_loss.append(loss) if batch_index % 100 == 0: print('Train Batch:{}, Train Loss:{:.4f}.'.format( batch_index, loss.item())) train_accuracy = model.evaluate_accuracy( train_pred.detach().numpy(), train_targets.detach().numpy()) print( 'Epoch:{}, Train Accuracy:{:.4f}, Train Mean loss:{:.4f}.'.format( epoch, train_accuracy, sum(train_loss) / len(train_loss))) print("\n") model.eval() with torch.no_grad(): for batch_index, (seqs, length, target) in enumerate(test_loader): seqs = seqs.type(torch.LongTensor) len_order = torch.argsort(length, descending=True) length = length[len_order] seqs = seqs[len_order] target = target[len_order].type(torch.LongTensor) seqs, target, length = seqs.to(device), target.to( device), length.to(device) output, pred_out = model(seqs, length, False) test_pred = torch.cat( (test_pred, pred_out.type(torch.float).cpu()), dim=0) test_targets = torch.cat( (test_targets, target.type(torch.float).cpu())) loss = criterion(output, target) test_loss.append(loss.item()) if batch_index % 100 == 0: print('Vali Batch:{}, Vali Loss:{:.4f}.'.format( batch_index, loss.item())) accuracy = model.evaluate_accuracy(test_pred.numpy(), test_targets.numpy()) print('Epoch:{}, Vali Accuracy:{:.4f}, Vali Mean loss:{:.4f}.'. format(epoch, accuracy, sum(test_loss) / len(test_loss))) # best save if accuracy > best_acc: best_acc = accuracy best_epoch = epoch torch.save(model.state_dict(), bert_lstm_save_path) # early stop if epoch - best_epoch >= patience: print('Early stopping') print('Best epoch: {}, Best accuracy: {:.4f}.'.format( best_epoch, best_acc)) print('\n\n') break model.load_state_dict(torch.load(bert_lstm_save_path)) model.eval() with torch.no_grad(): for batch_index, (seqs, length, target) in enumerate(test_loader): seqs = seqs.type(torch.LongTensor) len_order = torch.argsort(length, descending=True) length = length[len_order] seqs = seqs[len_order] target = target[len_order].type(torch.LongTensor) seqs, target, length = seqs.to(device), target.to( device), length.to(device) output, pred_out = model(seqs, length, False) test_pred = torch.cat( (test_pred, pred_out.type(torch.float).cpu()), dim=0) test_targets = torch.cat( (test_targets, target.type(torch.float).cpu())) loss = criterion(output, target) test_loss.append(loss.item()) accuracy = model.evaluate_accuracy(test_pred.numpy(), test_targets.numpy()) print('Test Accuracy:{:.4f}, Test Mean loss:{:.4f}.'.format( accuracy, sum(test_loss) / len(test_loss)))
def run(): args = parser.parse_args() data = args.data nlayer = args.nlayer file_path = args.file_path #'/content/drive/My Drive/Master_Final_Project/Genetic_attack/Code/nlp_adversarial_example_master_pytorch/glove.840B.300d.txt'#'/lustre/scratch/scratch/ucabdc3/lstm_attack' save_path = os.path.join(file_path, 'model_params') MAX_VOCAB_SIZE = 50000 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # with open(os.path.join(file_path, 'dataset_%d.pkl' %MAX_VOCAB_SIZE), 'rb') as f: # dataset = pickle.load(f) with open('aux_files/dataset_%d.pkl' % MAX_VOCAB_SIZE, 'rb') as f: dataset = pickle.load(f) # skip_list = np.load('aux_files/missed_embeddings_counter_%d.npy' %MAX_VOCAB_SIZE) embedding_matrix = np.load('aux_files/embeddings_glove_%d.npy' % (MAX_VOCAB_SIZE)) embedding_matrix = torch.tensor(embedding_matrix.T).to(device) # dist = np.load(('aux_files/dist_counter_%d.npy' %(MAX_VOCAB_SIZE))) # dist[0,:] = 100000 # dist[:,0] = 100000 # goog_lm = LM() # pytorch max_len = 100 # padded_train_raw = pad_sequences(dataset.train_seqs2, maxlen = max_len, padding = 'post') # padded_test_raw = pad_sequences(dataset.test_seqs2, maxlen = max_len, padding = 'post') # # TrainSet # data_set = Data_infor(padded_train_raw, dataset.train_y) # num_train = len(data_set) # indx = list(range(num_train)) # train_set = Subset(data_set, indx) if data.lower() == 'imdb': data_path = 'aclImdb' bert = BertModel.from_pretrained('bert-base-uncased') tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') data_processed = pre_processing(data_path, MAX_VOCAB_SIZE, max_len) tokenizer_select = args.tokenizer tokenizer_selection = tokenizer_select if tokenizer_selection.lower() != 'bert': data_processed.processing() train_sequences, test_sequences = data_processed.bert_indx(tokenizer) print('Self preprocessing') else: data_processed.bert_tokenize(tokenizer) train_sequences, test_sequences = data_processed.bert_indx(tokenizer) print('BERT tokenizer') train_text_init, test_text_init = data_processed.numerical( tokenizer, train_sequences, test_sequences) # train_text = pad_sequences(train_text_init, maxlen = max_len, padding = 'post') test_text = pad_sequences(test_text_init, maxlen=max_len, padding='post') # orig_test_text = pad_sequences(dataset.test_seqs2, maxlen = max_len, padding = 'post') # train_target = data_processed.all_train_labels test_target = data_processed.all_test_labels SAMPLE_SIZE = args.sample_size test_data, all_test_data = data_loading(test_text, test_target, SAMPLE_SIZE) # TestSet batch_size = 1 # data_set = Data_infor(padded_test_raw, dataset.test_y) # num_test = len(data_set) # indx = list(range(num_test)) # ## all_test_set = Subset(data_set, indx) # indx = random.sample(indx, SAMPLE_SIZE) # test_set = Subset(data_set, indx) # # test_loader = DataLoader(test_set, batch_size = batch_size, shuffle = False) test_loader_bert = DataLoader(test_data, batch_size=batch_size, shuffle=False) all_test_loader_bert = DataLoader(all_test_data, batch_size=128, shuffle=True) lstm_size = 128 rnn_state_save = os.path.join(save_path, 'best_bert_0.7_0.001_bert_200') model = bert_lstm( bert, 2, False, nlayer, lstm_size, True, 0.7 ) # batch_size=batch_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True) model.eval() model.load_state_dict(torch.load(rnn_state_save)) model = model.to(device) model.eval() test_pred = torch.tensor([]) test_targets = torch.tensor([]) with torch.no_grad(): for batch_index, (seqs, length, target) in enumerate(all_test_loader_bert): seqs = seqs.type(torch.LongTensor) len_order = torch.argsort(length, descending=True) length = length[len_order] seqs = seqs[len_order] target = target[len_order] seqs, target, length = seqs.to(device), target.to( device), length.to(device) output, pred_out = model.pred(seqs, length, False) test_pred = torch.cat((test_pred, pred_out.cpu()), dim=0) test_targets = torch.cat( (test_targets, target.type(torch.float).cpu())) accuracy = model.evaluate_accuracy(test_pred.numpy(), test_targets.numpy()) print('Test Accuracy:{:.4f}.'.format(accuracy)) # np.save(os.path.join(save_path,'accuracy.npy'), np.array(accuracy)) print('\n') n1 = 8 n2 = 4 pop_size = 60 max_iters = 20 n_prefix = 6 n_suffix = 6 batch_model = bert_lstm( bert, 2, False, nlayer, lstm_size, True, 0.7 ) #SentimentAnalysis(batch_size=pop_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True) batch_model.eval() batch_model.load_state_dict(torch.load(rnn_state_save)) batch_model.to(device) neighbour_model = bert_lstm( bert, 2, False, nlayer, lstm_size, True, 0.7 ) #SentimentAnalysis(batch_size=batch_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True) neighbour_model.eval() neighbour_model.load_state_dict(torch.load(rnn_state_save)) neighbour_model.to(device) lm_model = gpt_2_get_words_probs() ga_attack = GeneticAttack_pytorch(model, batch_model, neighbour_model, compute_dis, lm_model, tokenizer=tokenizer, max_iters=max_iters, dataset=dataset, pop_size=pop_size, n1=n1, n2=n2, n_prefix=n_prefix, n_suffix=n_suffix, use_lm=True, use_suffix=True) # TEST_SIZE = args.test_size # order_pre = 0 # n = 0 # seq_success = [] # seq_orig = [] # seq_orig_label = [] # word_varied = [] # # seq_success_path = os.path.join(save_path,'seq_success_perplexity_bert.npy') # # seq_orig_path = os.path.join(save_path,'seq_orig_perplexity_bert.npy') # # seq_orig_label_path = os.path.join(save_path,'seq_orig_label_perplexity_bert.npy') # # word_varied_path = os.path.join(save_path,'word_varied_perplexity_bert.npy') # # if order_pre != 0: # # seq_success = np.load(seq_success_path, allow_pickle = True).tolist() # # seq_orig = np.load(seq_orig_path).tolist() # # seq_orig_label = np.load(seq_orig_label_path).tolist() # # word_varied = np.load(word_varied_path, allow_pickle = True).tolist() # # n = len(seq_success) # for order, (seq, l, target) in enumerate(test_loader_bert): # if order>=order_pre: # seq_len = np.sum(np.sign(seq.numpy())) # seq = seq.type(torch.LongTensor) # seq, l = seq.to(device), l.to(device) # model.eval() # with torch.no_grad(): # orig_pred = np.argmax(model.pred(seq, l).cpu().detach().numpy()) # if orig_pred != target.numpy()[0]: # # print('Wrong original prediction') # # print('----------------------') # continue # if seq_len > 100: # # print('Sequence is too long') # # print('----------------------') # continue # print('Sequence number:{}'.format(order)) # print('Length of sentence: {}, Number of samples:{}'.format(l.item(), n+1)) # seq_orig.append(seq[0].cpu().detach().numpy()) # seq_orig_label.append(target.numpy()[0]) # target = int(1-target.numpy()[0]) # seq_success.append(ga_attack.attack(seq, target, l.type(torch.LongTensor))) # if None not in np.array(seq_success[n]): # w_be = [dataset.inv_dict[seq_orig[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])] # w_to = [dataset.inv_dict[seq_success[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])] # for i in range(len(w_be)): # print('{} ----> {}'.format(w_be[i], w_to[i])) # word_varied.append([w_be]+[w_to]) # else: # print('Fail') # print('----------------------') # n += 1 # np.save(seq_success_path, np.array(seq_success)) # np.save(seq_orig_path, np.array(seq_orig)) # np.save(seq_orig_label_path, np.array(seq_orig_label)) # np.save(word_varied_path, np.array(word_varied, dtype=object)) # if n>TEST_SIZE: # break TEST_SIZE = args.test_size order_pre = 0 n = 0 seq_success = [] seq_orig = [] seq_orig_label = [] word_varied = [] orig_list = [] adv_list = [] dist_list = [] # seq_success_path = os.path.join(save_path,'seq_success_perplexity_bert.npy') # seq_orig_path = os.path.join(save_path,'seq_orig_perplexity_bert.npy') # seq_orig_label_path = os.path.join(save_path,'seq_orig_label_perplexity_bert.npy') # word_varied_path = os.path.join(save_path,'word_varied_perplexity_bert.npy') # if order_pre != 0: # seq_success = np.load(seq_success_path, allow_pickle = True).tolist() # seq_orig = np.load(seq_orig_path).tolist() # seq_orig_label = np.load(seq_orig_label_path).tolist() # word_varied = np.load(word_varied_path, allow_pickle = True).tolist() # n = len(seq_success) for order, (seq, l, target) in enumerate(test_loader_bert): if order >= order_pre: seq_len = np.sum(np.sign(seq.numpy())) seq = seq.type(torch.LongTensor) seq, l = seq.to(device), l.to(device) model.eval() with torch.no_grad(): prediction = model.pred(seq, l, False)[1].cpu().detach().numpy() orig_pred = np.argmax(prediction) if orig_pred != target: # print('Wrong original prediction') # print('----------------------') continue if seq_len > 100: # print('Sequence is too long') # print('----------------------') continue print('Sequence number:{}'.format(order)) print('Predicted value:{}'.format(prediction)) print('Length of sentence: {}, Number of samples:{}'.format( l.item(), n + 1)) # seq_orig.append(seq[0].cpu().detach().numpy()) # seq_orig_label.append(target.numpy()[0]) target = int(1 - target) # seq_success.append(ga_attack.attack(seq, target, l.type(torch.LongTensor))) # if None not in np.array(seq_success[n]): # w_be = [dataset.inv_dict[seq_orig[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])] # w_to = [dataset.inv_dict[seq_success[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])] # for i in range(len(w_be)): # print('{} ----> {}'.format(w_be[i], w_to[i])) # word_varied.append([w_be]+[w_to]) # else: # print('Fail') # print('----------------------') # n += 1 # np.save(seq_success_path, np.array(seq_success)) # np.save(seq_orig_path, np.array(seq_orig)) # np.save(seq_orig_label_path, np.array(seq_orig_label)) # np.save(word_varied_path, np.array(word_varied, dtype=object)) # if n>TEST_SIZE: # break # orig_list.append(seq[0].cpu().detach().numpy()) x_adv, seq_out = ga_attack.attack(seq, target, l.type(torch.LongTensor)) orig_list.append(seq) adv_list.append(x_adv) if x_adv is None: print('%d failed' % (order)) dist_list.append(100000) else: num_changes = np.sum(np.array(seq_out) != np.array(x_adv)) print('%d - %d changed.' % (order, num_changes)) dist_list.append(num_changes) # display_utils.visualize_attack(sess, model, dataset, x_orig, x_adv) w_be = [ seq_out[i] for i in list( np.where(np.array(seq_out) != np.array(x_adv))[0]) ] w_to = [ x_adv[i] for i in list( np.where(np.array(seq_out) != np.array(x_adv))[0]) ] for i in range(len(w_be)): print('{} ----> {}'.format(w_be[i], w_to[i])) print('--------------------------') n += 1 if n > TEST_SIZE: break orig_len = [x.shape[1] for x in orig_list] normalized_dist_list = [ dist_list[i] / orig_len[i] for i in range(len(orig_list)) ] SUCCESS_THRESHOLD = 0.25 successful_attacks = [ x <= SUCCESS_THRESHOLD for x in normalized_dist_list ] print('Attack success rate : {:.2f}%'.format( np.mean(successful_attacks) * 100)) SUCCESS_THRESHOLD = 0.2 successful_attacks = [ x <= SUCCESS_THRESHOLD for x in normalized_dist_list ] print('Attack success rate : {:.2f}%'.format( np.mean(successful_attacks) * 100))
def run(): args = parser.parse_args() data = args.data nlayer = args.nlayer file_path = args.file_path save_path = os.path.join(file_path, 'model_params') MAX_VOCAB_SIZE = 50000 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") with open('aux_files_transfer/dataset_%d.pkl' %MAX_VOCAB_SIZE, 'rb') as f: dataset = pickle.load(f) # skip_list = np.load('aux_files/missed_embeddings_counter_%d.npy' %MAX_VOCAB_SIZE) embedding_matrix = np.load('aux_files_transfer/embeddings_glove_%d.npy' %(MAX_VOCAB_SIZE)) embedding_matrix = torch.tensor(embedding_matrix.T).to(device) # pytorch if data.lower() == 'imdb': data_path = 'aclImdb' bert = BertModel.from_pretrained('bert-base-uncased') tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') with open('attack_results_final_100_AL_200.pkl', 'rb') as f: results= pickle.load(f) seqs = [] lens = [] tgts = [] for i in range(len(results[1])): if np.array(results[1][i]).shape == (): continue seqs.append(' '.join([dataset.inv_dict[j] for j in results[1][i].tolist() if j!= 0])) lens.append(results[2][i]) tgts.append(results[3][i]) lens = torch.tensor(lens) tgts = torch.tensor(tgts) data_processed = pre_processing(seqs) tokenizer_select = args.tokenizer tokenizer_selection = tokenizer_select if tokenizer_selection.lower() != 'bert': data_processed.processing() train_sequences, test_sequences = data_processed.bert_indx(tokenizer) print('Self preprocessing') else: data_processed.bert_tokenize(tokenizer) test_sequences = data_processed.bert_indx(tokenizer) print('BERT tokenizer') test_text_init = data_processed.numerical(tokenizer,test_sequences) max_len = max([len(s) for s in test_text_init]) test_text = pad_sequences(test_text_init, maxlen = max_len, padding = 'post') all_test_data = TensorDataset(torch.tensor(test_text), lens, tgts) all_test_loader_bert = DataLoader(all_test_data, batch_size = 128, shuffle = True) lstm_size = 128 rnn_state_save = os.path.join(save_path,'best_bert_0.7_0.001_bert_100') model = bert_lstm(bert, 2, False, nlayer, lstm_size, True, 0.7)# batch_size=batch_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True) model.eval() model.load_state_dict(torch.load(rnn_state_save)) model = model.to(device) model.eval() test_pred = torch.tensor([]) test_targets = torch.tensor([]) with torch.no_grad(): for batch_index, (seqs, length, target) in enumerate(all_test_loader_bert): seqs = seqs.type(torch.LongTensor) len_order = torch.argsort(length, descending = True) length = length[len_order] seqs = seqs[len_order] target = target[len_order] seqs, target, length = seqs.to(device), target.to(device), length.to(device) output, pred_out = model.pred(seqs, length, False) test_pred = torch.cat((test_pred, pred_out.cpu()), dim = 0) test_targets = torch.cat((test_targets, target.type(torch.float).cpu())) accuracy = model.evaluate_accuracy(test_pred.numpy(), test_targets.numpy()) print('Test Accuracy:{:.4f}.'.format(accuracy))