def imdb_run():
args = parser.parse_args()
data = args.data
freeze = args.freeze
nlayer = args.nlayer
kept_prob = args.kept_prob_dropout
bert_lstm_save_path = args.save_path
learning_rate = args.learning_rate
epoches = args.epoches
tokenizer_selection = args.tokenizer
if data.lower() == 'imdb':
data_path = 'aclImdb'
bert = BertModel.from_pretrained('bert-base-uncased')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
max_len = 100
# max_vocab = bert.config.to_dict()['vocab_size']-3
# data_processed = pre_processing(data_path, max_vocab)
# train_sequences, test_sequences = data_processed.seqs_num()
# train_text_init, test_text_init = data_processed.numerical(train_sequences, test_sequences, max_len = max_len)
max_vocab = 50000
data_processed = pre_processing(data_path, max_vocab, max_len)
if tokenizer_selection.lower() != 'bert':
data_processed.processing()
train_sequences, test_sequences = data_processed.bert_indx(tokenizer)
print('Self preprocessing')
else:
data_processed.bert_tokenize(tokenizer)
train_sequences, test_sequences = data_processed.bert_indx(tokenizer)
print('BERT tokenizer')
train_text_init, test_text_init = data_processed.numerical(
tokenizer, train_sequences, test_sequences)
train_text = pad_sequences(train_text_init, maxlen=max_len, padding='post')
test_text = pad_sequences(test_text_init, maxlen=max_len, padding='post')
train_target = data_processed.all_train_labels
test_target = data_processed.all_test_labels
all_train_data, train_data, vali_data, test_data = data_loading(
train_text, test_text, train_target, test_target)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device)
BatchSize = 128 #int(length_train/200)
all_train_loader = DataLoader(all_train_data,
batch_size=BatchSize,
shuffle=True)
train_loader = DataLoader(train_data, batch_size=BatchSize, shuffle=True)
vali_loader = DataLoader(vali_data, batch_size=BatchSize, shuffle=True)
test_loader = DataLoader(test_data, batch_size=BatchSize, shuffle=True)
bidirection = args.bidirection
model = bert_lstm(bert, 2, bidirection, nlayer, 128, freeze, kept_prob)
model.to(device)
criterion = nn.CrossEntropyLoss()
optimiser = torch.optim.AdamW(
[cont for cont in model.parameters() if cont.requires_grad],
lr=learning_rate)
bert_lstm_save_path = os.path.join(
bert_lstm_save_path,
'best_bert_' + str(kept_prob) + '_' + str(learning_rate) + '_' +
str(tokenizer_selection) + '_' + str(max_len))
best_epoch = 0
best_acc = 0
patience = 20
for epoch in range(epoches):
test_pred = torch.tensor([])
test_targets = torch.tensor([])
train_pred = torch.tensor([])
train_targets = torch.tensor([])
test_loss = []
train_loss = []
model.train()
for batch_index, (seqs, length, target) in enumerate(all_train_loader):
seqs = seqs.type(torch.LongTensor)
args = torch.argsort(length, descending=True)
length = length[args]
seqs = seqs[args][:, 0:length[0]]
target = target[args].type(torch.LongTensor)
optimiser.zero_grad()
seqs, target, length = seqs.to(device), target.to(
device), length.to(device)
output, pred_out = model(seqs, length, True)
loss = criterion(output, target)
loss.backward()
optimiser.step()
train_pred = torch.cat((train_pred, pred_out.cpu()), dim=0)
train_targets = torch.cat(
(train_targets, target.type(torch.float).cpu()))
train_loss.append(loss)
if batch_index % 100 == 0:
print('Train Batch:{}, Train Loss:{:.4f}.'.format(
batch_index, loss.item()))
train_accuracy = model.evaluate_accuracy(
train_pred.detach().numpy(),
train_targets.detach().numpy())
print(
'Epoch:{}, Train Accuracy:{:.4f}, Train Mean loss:{:.4f}.'.format(
epoch, train_accuracy,
sum(train_loss) / len(train_loss)))
print("\n")
model.eval()
with torch.no_grad():
for batch_index, (seqs, length, target) in enumerate(test_loader):
seqs = seqs.type(torch.LongTensor)
len_order = torch.argsort(length, descending=True)
length = length[len_order]
seqs = seqs[len_order]
target = target[len_order].type(torch.LongTensor)
seqs, target, length = seqs.to(device), target.to(
device), length.to(device)
output, pred_out = model(seqs, length, False)
test_pred = torch.cat(
(test_pred, pred_out.type(torch.float).cpu()), dim=0)
test_targets = torch.cat(
(test_targets, target.type(torch.float).cpu()))
loss = criterion(output, target)
test_loss.append(loss.item())
if batch_index % 100 == 0:
print('Vali Batch:{}, Vali Loss:{:.4f}.'.format(
batch_index, loss.item()))
accuracy = model.evaluate_accuracy(test_pred.numpy(),
test_targets.numpy())
print('Epoch:{}, Vali Accuracy:{:.4f}, Vali Mean loss:{:.4f}.'.
format(epoch, accuracy,
sum(test_loss) / len(test_loss)))
# best save
if accuracy > best_acc:
best_acc = accuracy
best_epoch = epoch
torch.save(model.state_dict(), bert_lstm_save_path)
# early stop
if epoch - best_epoch >= patience:
print('Early stopping')
print('Best epoch: {}, Best accuracy: {:.4f}.'.format(
best_epoch, best_acc))
print('\n\n')
break
model.load_state_dict(torch.load(bert_lstm_save_path))
model.eval()
with torch.no_grad():
for batch_index, (seqs, length, target) in enumerate(test_loader):
seqs = seqs.type(torch.LongTensor)
len_order = torch.argsort(length, descending=True)
length = length[len_order]
seqs = seqs[len_order]
target = target[len_order].type(torch.LongTensor)
seqs, target, length = seqs.to(device), target.to(
device), length.to(device)
output, pred_out = model(seqs, length, False)
test_pred = torch.cat(
(test_pred, pred_out.type(torch.float).cpu()), dim=0)
test_targets = torch.cat(
(test_targets, target.type(torch.float).cpu()))
loss = criterion(output, target)
test_loss.append(loss.item())
accuracy = model.evaluate_accuracy(test_pred.numpy(),
test_targets.numpy())
print('Test Accuracy:{:.4f}, Test Mean loss:{:.4f}.'.format(
accuracy,
sum(test_loss) / len(test_loss)))
def run():
args = parser.parse_args()
data = args.data
nlayer = args.nlayer
file_path = args.file_path #'/content/drive/My Drive/Master_Final_Project/Genetic_attack/Code/nlp_adversarial_example_master_pytorch/glove.840B.300d.txt'#'/lustre/scratch/scratch/ucabdc3/lstm_attack'
save_path = os.path.join(file_path, 'model_params')
MAX_VOCAB_SIZE = 50000
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# with open(os.path.join(file_path, 'dataset_%d.pkl' %MAX_VOCAB_SIZE), 'rb') as f:
# dataset = pickle.load(f)
with open('aux_files/dataset_%d.pkl' % MAX_VOCAB_SIZE, 'rb') as f:
dataset = pickle.load(f)
# skip_list = np.load('aux_files/missed_embeddings_counter_%d.npy' %MAX_VOCAB_SIZE)
embedding_matrix = np.load('aux_files/embeddings_glove_%d.npy' %
(MAX_VOCAB_SIZE))
embedding_matrix = torch.tensor(embedding_matrix.T).to(device)
# dist = np.load(('aux_files/dist_counter_%d.npy' %(MAX_VOCAB_SIZE)))
# dist[0,:] = 100000
# dist[:,0] = 100000
# goog_lm = LM()
# pytorch
max_len = 100
# padded_train_raw = pad_sequences(dataset.train_seqs2, maxlen = max_len, padding = 'post')
# padded_test_raw = pad_sequences(dataset.test_seqs2, maxlen = max_len, padding = 'post')
# # TrainSet
# data_set = Data_infor(padded_train_raw, dataset.train_y)
# num_train = len(data_set)
# indx = list(range(num_train))
# train_set = Subset(data_set, indx)
if data.lower() == 'imdb':
data_path = 'aclImdb'
bert = BertModel.from_pretrained('bert-base-uncased')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
data_processed = pre_processing(data_path, MAX_VOCAB_SIZE, max_len)
tokenizer_select = args.tokenizer
tokenizer_selection = tokenizer_select
if tokenizer_selection.lower() != 'bert':
data_processed.processing()
train_sequences, test_sequences = data_processed.bert_indx(tokenizer)
print('Self preprocessing')
else:
data_processed.bert_tokenize(tokenizer)
train_sequences, test_sequences = data_processed.bert_indx(tokenizer)
print('BERT tokenizer')
train_text_init, test_text_init = data_processed.numerical(
tokenizer, train_sequences, test_sequences)
# train_text = pad_sequences(train_text_init, maxlen = max_len, padding = 'post')
test_text = pad_sequences(test_text_init, maxlen=max_len, padding='post')
# orig_test_text = pad_sequences(dataset.test_seqs2, maxlen = max_len, padding = 'post')
# train_target = data_processed.all_train_labels
test_target = data_processed.all_test_labels
SAMPLE_SIZE = args.sample_size
test_data, all_test_data = data_loading(test_text, test_target,
SAMPLE_SIZE)
# TestSet
batch_size = 1
# data_set = Data_infor(padded_test_raw, dataset.test_y)
# num_test = len(data_set)
# indx = list(range(num_test))
#
## all_test_set = Subset(data_set, indx)
# indx = random.sample(indx, SAMPLE_SIZE)
# test_set = Subset(data_set, indx)
#
# test_loader = DataLoader(test_set, batch_size = batch_size, shuffle = False)
test_loader_bert = DataLoader(test_data,
batch_size=batch_size,
shuffle=False)
all_test_loader_bert = DataLoader(all_test_data,
batch_size=128,
shuffle=True)
lstm_size = 128
rnn_state_save = os.path.join(save_path, 'best_bert_0.7_0.001_bert_200')
model = bert_lstm(
bert, 2, False, nlayer, lstm_size, True, 0.7
) # batch_size=batch_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True)
model.eval()
model.load_state_dict(torch.load(rnn_state_save))
model = model.to(device)
model.eval()
test_pred = torch.tensor([])
test_targets = torch.tensor([])
with torch.no_grad():
for batch_index, (seqs, length,
target) in enumerate(all_test_loader_bert):
seqs = seqs.type(torch.LongTensor)
len_order = torch.argsort(length, descending=True)
length = length[len_order]
seqs = seqs[len_order]
target = target[len_order]
seqs, target, length = seqs.to(device), target.to(
device), length.to(device)
output, pred_out = model.pred(seqs, length, False)
test_pred = torch.cat((test_pred, pred_out.cpu()), dim=0)
test_targets = torch.cat(
(test_targets, target.type(torch.float).cpu()))
accuracy = model.evaluate_accuracy(test_pred.numpy(),
test_targets.numpy())
print('Test Accuracy:{:.4f}.'.format(accuracy))
# np.save(os.path.join(save_path,'accuracy.npy'), np.array(accuracy))
print('\n')
n1 = 8
n2 = 4
pop_size = 60
max_iters = 20
n_prefix = 6
n_suffix = 6
batch_model = bert_lstm(
bert, 2, False, nlayer, lstm_size, True, 0.7
) #SentimentAnalysis(batch_size=pop_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True)
batch_model.eval()
batch_model.load_state_dict(torch.load(rnn_state_save))
batch_model.to(device)
neighbour_model = bert_lstm(
bert, 2, False, nlayer, lstm_size, True, 0.7
) #SentimentAnalysis(batch_size=batch_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True)
neighbour_model.eval()
neighbour_model.load_state_dict(torch.load(rnn_state_save))
neighbour_model.to(device)
lm_model = gpt_2_get_words_probs()
ga_attack = GeneticAttack_pytorch(model,
batch_model,
neighbour_model,
compute_dis,
lm_model,
tokenizer=tokenizer,
max_iters=max_iters,
dataset=dataset,
pop_size=pop_size,
n1=n1,
n2=n2,
n_prefix=n_prefix,
n_suffix=n_suffix,
use_lm=True,
use_suffix=True)
# TEST_SIZE = args.test_size
# order_pre = 0
# n = 0
# seq_success = []
# seq_orig = []
# seq_orig_label = []
# word_varied = []
# # seq_success_path = os.path.join(save_path,'seq_success_perplexity_bert.npy')
# # seq_orig_path = os.path.join(save_path,'seq_orig_perplexity_bert.npy')
# # seq_orig_label_path = os.path.join(save_path,'seq_orig_label_perplexity_bert.npy')
# # word_varied_path = os.path.join(save_path,'word_varied_perplexity_bert.npy')
# # if order_pre != 0:
# # seq_success = np.load(seq_success_path, allow_pickle = True).tolist()
# # seq_orig = np.load(seq_orig_path).tolist()
# # seq_orig_label = np.load(seq_orig_label_path).tolist()
# # word_varied = np.load(word_varied_path, allow_pickle = True).tolist()
# # n = len(seq_success)
# for order, (seq, l, target) in enumerate(test_loader_bert):
# if order>=order_pre:
# seq_len = np.sum(np.sign(seq.numpy()))
# seq = seq.type(torch.LongTensor)
# seq, l = seq.to(device), l.to(device)
# model.eval()
# with torch.no_grad():
# orig_pred = np.argmax(model.pred(seq, l).cpu().detach().numpy())
# if orig_pred != target.numpy()[0]:
# # print('Wrong original prediction')
# # print('----------------------')
# continue
# if seq_len > 100:
# # print('Sequence is too long')
# # print('----------------------')
# continue
# print('Sequence number:{}'.format(order))
# print('Length of sentence: {}, Number of samples:{}'.format(l.item(), n+1))
# seq_orig.append(seq[0].cpu().detach().numpy())
# seq_orig_label.append(target.numpy()[0])
# target = int(1-target.numpy()[0])
# seq_success.append(ga_attack.attack(seq, target, l.type(torch.LongTensor)))
# if None not in np.array(seq_success[n]):
# w_be = [dataset.inv_dict[seq_orig[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])]
# w_to = [dataset.inv_dict[seq_success[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])]
# for i in range(len(w_be)):
# print('{} ----> {}'.format(w_be[i], w_to[i]))
# word_varied.append([w_be]+[w_to])
# else:
# print('Fail')
# print('----------------------')
# n += 1
# np.save(seq_success_path, np.array(seq_success))
# np.save(seq_orig_path, np.array(seq_orig))
# np.save(seq_orig_label_path, np.array(seq_orig_label))
# np.save(word_varied_path, np.array(word_varied, dtype=object))
# if n>TEST_SIZE:
# break
TEST_SIZE = args.test_size
order_pre = 0
n = 0
seq_success = []
seq_orig = []
seq_orig_label = []
word_varied = []
orig_list = []
adv_list = []
dist_list = []
# seq_success_path = os.path.join(save_path,'seq_success_perplexity_bert.npy')
# seq_orig_path = os.path.join(save_path,'seq_orig_perplexity_bert.npy')
# seq_orig_label_path = os.path.join(save_path,'seq_orig_label_perplexity_bert.npy')
# word_varied_path = os.path.join(save_path,'word_varied_perplexity_bert.npy')
# if order_pre != 0:
# seq_success = np.load(seq_success_path, allow_pickle = True).tolist()
# seq_orig = np.load(seq_orig_path).tolist()
# seq_orig_label = np.load(seq_orig_label_path).tolist()
# word_varied = np.load(word_varied_path, allow_pickle = True).tolist()
# n = len(seq_success)
for order, (seq, l, target) in enumerate(test_loader_bert):
if order >= order_pre:
seq_len = np.sum(np.sign(seq.numpy()))
seq = seq.type(torch.LongTensor)
seq, l = seq.to(device), l.to(device)
model.eval()
with torch.no_grad():
prediction = model.pred(seq, l,
False)[1].cpu().detach().numpy()
orig_pred = np.argmax(prediction)
if orig_pred != target:
# print('Wrong original prediction')
# print('----------------------')
continue
if seq_len > 100:
# print('Sequence is too long')
# print('----------------------')
continue
print('Sequence number:{}'.format(order))
print('Predicted value:{}'.format(prediction))
print('Length of sentence: {}, Number of samples:{}'.format(
l.item(), n + 1))
# seq_orig.append(seq[0].cpu().detach().numpy())
# seq_orig_label.append(target.numpy()[0])
target = int(1 - target)
# seq_success.append(ga_attack.attack(seq, target, l.type(torch.LongTensor)))
# if None not in np.array(seq_success[n]):
# w_be = [dataset.inv_dict[seq_orig[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])]
# w_to = [dataset.inv_dict[seq_success[n][i]] for i in list(np.where(seq_success[n] != seq_orig[n])[0])]
# for i in range(len(w_be)):
# print('{} ----> {}'.format(w_be[i], w_to[i]))
# word_varied.append([w_be]+[w_to])
# else:
# print('Fail')
# print('----------------------')
# n += 1
# np.save(seq_success_path, np.array(seq_success))
# np.save(seq_orig_path, np.array(seq_orig))
# np.save(seq_orig_label_path, np.array(seq_orig_label))
# np.save(word_varied_path, np.array(word_varied, dtype=object))
# if n>TEST_SIZE:
# break
# orig_list.append(seq[0].cpu().detach().numpy())
x_adv, seq_out = ga_attack.attack(seq, target,
l.type(torch.LongTensor))
orig_list.append(seq)
adv_list.append(x_adv)
if x_adv is None:
print('%d failed' % (order))
dist_list.append(100000)
else:
num_changes = np.sum(np.array(seq_out) != np.array(x_adv))
print('%d - %d changed.' % (order, num_changes))
dist_list.append(num_changes)
# display_utils.visualize_attack(sess, model, dataset, x_orig, x_adv)
w_be = [
seq_out[i] for i in list(
np.where(np.array(seq_out) != np.array(x_adv))[0])
]
w_to = [
x_adv[i] for i in list(
np.where(np.array(seq_out) != np.array(x_adv))[0])
]
for i in range(len(w_be)):
print('{} ----> {}'.format(w_be[i], w_to[i]))
print('--------------------------')
n += 1
if n > TEST_SIZE:
break
orig_len = [x.shape[1] for x in orig_list]
normalized_dist_list = [
dist_list[i] / orig_len[i] for i in range(len(orig_list))
]
SUCCESS_THRESHOLD = 0.25
successful_attacks = [
x <= SUCCESS_THRESHOLD for x in normalized_dist_list
]
print('Attack success rate : {:.2f}%'.format(
np.mean(successful_attacks) * 100))
SUCCESS_THRESHOLD = 0.2
successful_attacks = [
x <= SUCCESS_THRESHOLD for x in normalized_dist_list
]
print('Attack success rate : {:.2f}%'.format(
np.mean(successful_attacks) * 100))
def run():
args = parser.parse_args()
data = args.data
nlayer = args.nlayer
file_path = args.file_path
save_path = os.path.join(file_path, 'model_params')
MAX_VOCAB_SIZE = 50000
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
with open('aux_files_transfer/dataset_%d.pkl' %MAX_VOCAB_SIZE, 'rb') as f:
dataset = pickle.load(f)
# skip_list = np.load('aux_files/missed_embeddings_counter_%d.npy' %MAX_VOCAB_SIZE)
embedding_matrix = np.load('aux_files_transfer/embeddings_glove_%d.npy' %(MAX_VOCAB_SIZE))
embedding_matrix = torch.tensor(embedding_matrix.T).to(device)
# pytorch
if data.lower() == 'imdb':
data_path = 'aclImdb'
bert = BertModel.from_pretrained('bert-base-uncased')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
with open('attack_results_final_100_AL_200.pkl', 'rb') as f:
results= pickle.load(f)
seqs = []
lens = []
tgts = []
for i in range(len(results[1])):
if np.array(results[1][i]).shape == ():
continue
seqs.append(' '.join([dataset.inv_dict[j] for j in results[1][i].tolist() if j!= 0]))
lens.append(results[2][i])
tgts.append(results[3][i])
lens = torch.tensor(lens)
tgts = torch.tensor(tgts)
data_processed = pre_processing(seqs)
tokenizer_select = args.tokenizer
tokenizer_selection = tokenizer_select
if tokenizer_selection.lower() != 'bert':
data_processed.processing()
train_sequences, test_sequences = data_processed.bert_indx(tokenizer)
print('Self preprocessing')
else:
data_processed.bert_tokenize(tokenizer)
test_sequences = data_processed.bert_indx(tokenizer)
print('BERT tokenizer')
test_text_init = data_processed.numerical(tokenizer,test_sequences)
max_len = max([len(s) for s in test_text_init])
test_text = pad_sequences(test_text_init, maxlen = max_len, padding = 'post')
all_test_data = TensorDataset(torch.tensor(test_text), lens, tgts)
all_test_loader_bert = DataLoader(all_test_data, batch_size = 128, shuffle = True)
lstm_size = 128
rnn_state_save = os.path.join(save_path,'best_bert_0.7_0.001_bert_100')
model = bert_lstm(bert, 2, False, nlayer, lstm_size, True, 0.7)# batch_size=batch_size, embedding_matrix = embedding_matrix, hidden_size = lstm_size, kept_prob = 0.73, num_layers=2, bidirection=True)
model.eval()
model.load_state_dict(torch.load(rnn_state_save))
model = model.to(device)
model.eval()
test_pred = torch.tensor([])
test_targets = torch.tensor([])
with torch.no_grad():
for batch_index, (seqs, length, target) in enumerate(all_test_loader_bert):
seqs = seqs.type(torch.LongTensor)
len_order = torch.argsort(length, descending = True)
length = length[len_order]
seqs = seqs[len_order]
target = target[len_order]
seqs, target, length = seqs.to(device), target.to(device), length.to(device)
output, pred_out = model.pred(seqs, length, False)
test_pred = torch.cat((test_pred, pred_out.cpu()), dim = 0)
test_targets = torch.cat((test_targets, target.type(torch.float).cpu()))
accuracy = model.evaluate_accuracy(test_pred.numpy(), test_targets.numpy())
print('Test Accuracy:{:.4f}.'.format(accuracy))