def train_Seq2Seq(train_data, test_data, model, criterion, optimizer, cur_dir,
attack_vocab):
best_accuracy = 0.0
for epoch in range(AttackConfig.epochs):
logging(f'epoch {epoch} start')
logging(f'epoch {epoch} train Seq2Seq model')
model.train()
loss_mean = 0.0
n = 0
for x, x_mask, y, _ in train_data:
x, x_mask, y = x.to(AttackConfig.train_device), x_mask.to(
AttackConfig.train_device), y.to(AttackConfig.train_device)
model.zero_grad()
logits = model(x, x_mask, is_noise=False)
logits = logits.reshape(-1, logits.shape[-1])
y = y.reshape(-1)
loss = criterion(logits, y)
loss_mean += loss.item()
loss.backward()
optimizer.step()
n += (x.shape[0] * x.shape[1])
logging(f"epoch {epoch} train_loss is {loss_mean / n}")
eval_accuracy = evaluate_Seq2Seq(
test_data, model, cur_dir + f'/eval_Seq2Seq_model_epoch_{epoch}',
attack_vocab)
logging(f"epoch {epoch} test_acc is {eval_accuracy}")
if best_accuracy < eval_accuracy:
best_accuracy = eval_accuracy
logging('Saveing Seq2Seq models...')
torch.save(model.state_dict(), cur_dir + r'/Seq2Seq_model.pt')
def token2idx(self):
logging(f'{self.path} in token2idx')
for tokens in self.data_tokens:
self.data_idx.append(self.tokenizer.convert_tokens_to_ids(tokens))
self.data_mask.append([1] * len(tokens))
if self.attack_vocab:
for tokens in self.label_tokens:
self.label_idx.append(
[self.attack_vocab.get_index(token) for token in tokens])
else:
for tokens in self.label_tokens:
self.label_idx.append(
self.tokenizer.convert_tokens_to_ids(tokens))
for i in range(len(self.data_idx)):
if len(self.data_idx[i]) < self.sen_len:
self.data_idx[i] += [0
] * (self.sen_len - len(self.data_idx[i]))
self.data_mask[i] += [0] * (self.sen_len -
len(self.data_mask[i]))
if len(self.label_idx[i]) < self.sen_len:
self.label_idx[i] += [0] * (self.sen_len -
len(self.label_idx[i]))
def write_syn_csv(self, path):
with open(path, 'w', newline='', encoding='utf-8') as file:
writer = csv.writer(file, delimiter=',')
for key, value in self.syn_dict.items():
writer.writerow([key] + list(value))
logging(f'write synoymous words {self.syn_count} to {path}')
def read_standard_data(self, path, sentences, debug_mode=False):
label_classes = SNLIConfig.label_classes
premise_data = []
hypothesis_data = []
labels = []
if debug_mode:
i = 1000
with open(path, 'r', encoding='utf-8') as file:
for line in file:
tokens = line.strip().split('\t')
labels.append(label_classes[tokens[0].strip()])
premise_data.append(sentences[int(tokens[1].strip())])
hypothesis_data.append(sentences[int(tokens[2].strip())])
i -= 1
if i == 0:
break
logging(f'loading data {len(premise_data)} from {path}')
return premise_data, hypothesis_data, labels
with open(path, 'r', encoding='utf-8') as file:
for line in file:
tokens = line.strip().split('\t')
labels.append(label_classes[tokens[0].strip()])
premise_data.append(sentences[int(tokens[1].strip())])
hypothesis_data.append(sentences[int(tokens[2].strip())])
logging(f'loading data {len(premise_data)} from {path}')
return premise_data, hypothesis_data, labels
def data2tokens(self):
logging(f'{self.path} in data2tokens')
for sen in self.premise_data:
data_tokens = ['[CLS]']
data_tokens += self.tokenizer.tokenize(sen)[:self.sen_len - 1]
if self.attack_vocab:
label_tokens = self.baseline_tokenizer(sen)[:self.sen_len - 1]
label_tokens += ['[SEP]']
else:
label_tokens = self.tokenizer.tokenize(sen)[:self.sen_len - 1]
label_tokens += ['[SEP]']
self.premise_data_tokens.append(data_tokens)
self.premise_label_tokens.append(label_tokens)
for sen in self.hypothesis_data:
data_tokens = ['[CLS]']
data_tokens += self.tokenizer.tokenize(sen)[:self.sen_len - 1]
if self.attack_vocab:
label_tokens = self.baseline_tokenizer(sen)[:self.sen_len - 1]
label_tokens += ['[SEP]']
else:
label_tokens = self.tokenizer.tokenize(sen)[:self.sen_len - 1]
label_tokens += ['[SEP]']
self.hypothesis_data_tokens.append(data_tokens)
self.hypothesis_label_tokens.append(label_tokens)
def write_results_to_file(models, results, logs):
with open(args.save_path, 'w', newline='', encoding='utf-8') as file:
writer = csv.writer(file, delimiter=',')
writer.writerow([
'model_name', 'dataset', 'origin_acc', 'clean_acc', 'adv_acc',
'acc_shift', 'success_rate', 'sub_rate'
])
for model, res, log in zip(models, results, logs):
print(
f'{model} {dataset_name} testdata {len(test_data)} all acc is {res[0]:.5f}'
)
print(
f'{model} {dataset_name} cleandata {len(clean_data)} acc is {res[1]:.5f}'
)
print(
f'{model} {dataset_name} advdata {len(clean_data)} acc is {res[2]:.5f}'
)
print(
f'{model} {dataset_name} advdata {log[0]}, mean sub_rate {log[1]:.5f}, mean NE_rate {log[2]:.5f}'
)
acc_shift = res[1] - res[2]
success_rate = acc_shift / res[1]
writer.writerow([
model, dataset_name, res[0], res[1], res[2], acc_shift,
success_rate, log[1]
])
logging(f'evaluate results saved in {args.save_path}')
def read_syn_csv(self, path):
with open(path, 'r', newline='', encoding='utf-8') as file:
reader = csv.reader(file, delimiter=',')
for line in reader:
res = set(line[1:])
self.syn_count += len(res)
self.syn_dict[line[0]] = res
logging(
f'load syn_words from {path}, key word is {len(self.syn_dict)}, synonymous words is {self.syn_count}'
)
def token2seq(self, vocab:'Vocab', maxlen:int):
if len(self.data_seq) > 0:
self.data_seq.clear()
self.labels_tensor.clear()
logging(f'data is train {self.is_train} is to sequence!')
self.vocab = vocab
self.maxlen = maxlen
assert self.data_token is not None
for tokens in self.data_token:
self.data_seq.append(self.__encode_tokens(tokens))
for label in self.labels:
self.labels_tensor.append(torch.tensor(label))
def build_syn_dict(self, vocab: 'Vocab', path):
assert len(vocab.word_dict) > 0
for key, value in tqdm(vocab.word_dict.items()):
if value == 0: continue
res = self.get_similarity_words(key)
if len(res) > 0: self.syn_dict[key] = res
self.syn_count += len(res)
num = len(self.syn_dict)
logging(
f'synonymous words has been built, key word is {num}, synonymous words is {self.syn_count}'
)
self.write_syn_csv(path)
def data2tokens(self):
logging(f'{self.path} in data2tokens')
for sen in self.datas:
if self.attack_vocab:
data_tokens = ['[CLS]']
data_tokens += self.baseline_tokenizer(sen)[:self.sen_len - 1]
label_tokens = self.baseline_tokenizer(sen)[:self.sen_len - 1]
label_tokens += ['[SEP]']
else:
data_tokens = ['[CLS]']
data_tokens += self.tokenizer.tokenize(sen)[:self.sen_len - 1]
label_tokens = self.tokenizer.tokenize(sen)[:self.sen_len - 1]
label_tokens += ['[SEP]']
self.data_tokens.append(data_tokens)
self.label_tokens.append(label_tokens)
def evaluate_Seq2Seq(test_data, Seq2Seq_model, dir, attack_vocab):
Seq2Seq_model.eval()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
logging(f'Saving evaluate of Seq2Seq_model outputs into {dir}')
with torch.no_grad():
acc_sum = 0
n = 0
for x, x_mask, y, _ in test_data:
x, x_mask, y = x.to(AttackConfig.train_device), x_mask.to(
AttackConfig.train_device), y.to(AttackConfig.train_device)
logits = Seq2Seq_model(x, x_mask, is_noise=False)
# outputs_idx: [batch, sen_len]
outputs_idx = logits.argmax(dim=2)
acc_sum += (outputs_idx == y).float().sum().item()
n += y.shape[0] * y.shape[1]
if attack_vocab:
with open(dir, 'a') as f:
for i in range(len(y)):
f.write('-------orginal sentence----------\n')
f.write(' '.join(
[attack_vocab.get_word(token)
for token in y[i]]) + '\n')
f.write(
'-------sentence -> encoder -> decoder----------\n'
)
f.write(' '.join([
attack_vocab.get_word(token)
for token in outputs_idx[i]
]) + '\n' * 2)
else:
with open(dir, 'a') as f:
for i in range(len(y)):
f.write('-------orginal sentence----------\n')
f.write(
' '.join(tokenizer.convert_ids_to_tokens(y[i])) +
'\n')
f.write(
'-------sentence -> encoder -> decoder----------\n'
)
f.write(' '.join(
tokenizer.convert_ids_to_tokens(outputs_idx[i])) +
'\n' * 2)
return acc_sum / n
def __init__(self, tokenizer_type='normal', remove_stop_words=False):
self.is_remove_stop_words = remove_stop_words
if tokenizer_type == 'normal':
self.tokenizer = self.normal_token
elif tokenizer_type == 'spacy':
self.nlp = spacy.load('en_core_web_sm')
self.tokenizer = self.spacy_token
else:
raise RuntimeError(
f'Tokenizer type is error, do not have type {tokenizer_type}')
self.token_type = tokenizer_type
self.stop_words = set(stopwords.words('english'))
for w in [
"<br />", '!', ',', '.', '?', '-s', '-ly', '</s>', 's', '</',
'>', '/>', 'br', '<'
]:
self.stop_words.add(w)
logging(
f'using tokenizer {tokenizer_type}, is_remove_stop_words={remove_stop_words}'
)
def evaluate(now_ep) -> float:
test_loss = 0.0
logging('starting evaluate!')
net.eval()
with torch.no_grad():
correct = 0
total = 0
for (X, label) in tqdm(test_data):
X, label = X.to(config_device), label.to(config_device)
logits = net(X)
test_loss += criterion(logits, label).item()
predict = logits.argmax(dim=1)
correct += predict.eq(label).float().sum().item()
total += X.size(0)
correct /= total
test_loss /= len(test_data)
logging(
f'epoch {now_ep} evaluate done! test acc {correct:.5f}, best acc{best_acc:.5f}, test batch loss {test_loss:.5f}'
)
return correct, test_loss
def read_standard_data(self, path, debug_mode=False):
path_list = []
logging(f'start loading data from {path}')
dirs = os.listdir(path)
for dir in dirs:
if dir == 'pos' or dir == 'neg':
file_list = os.listdir(os.path.join(path, dir))
file_list = map(lambda x: os.path.join(path, dir, x),
file_list)
path_list += list(file_list)
datas = []
labels = []
if debug_mode:
i = 1000
for p in path_list:
label = 0 if 'neg' in p else 1
with open(p, 'r', encoding='utf-8') as file:
datas.append(file.readline())
labels.append(label)
i -= 1
if i == 0:
logging(f'loading data {len(datas)} from {path}')
return datas, labels
for p in path_list:
label = 0 if 'neg' in p else 1
with open(p, 'r', encoding='utf-8') as file:
datas.append(file.readline())
labels.append(label)
logging(f'loading data {len(datas)} from {path}')
return datas, labels
def evaluate(dataset: MyDataset, net: Module):
assert dataset.is_train == False
assert len(dataset) > 0
dataset = DataLoader(dataset, batch_size=1000, shuffle=False)
test_loss = 0.0
net.eval()
logging('starting evaluate!')
with torch.no_grad():
correct = 0
total = 0
for (X, label) in tqdm(dataset):
X, label = X.to(config_device), label.to(config_device)
logits = net(X)
predict = logits.argmax(dim=1)
correct += predict.eq(label).float().sum().item()
total += X.size(0)
correct /= total
test_loss /= len(test_data)
logging(
f'evaluate done! acc {correct:.5f}, test average loss {test_loss:.5f}')
return correct
def read_standard_data(self, path, debug_mode=False):
data = []
labels = []
if debug_mode:
i = 1000
with open(path, 'r', encoding='utf-8') as file:
for line in file:
i -= 1
line = line.strip('\n')
data.append(line[:-1])
labels.append(int(line[-1]))
if i == 0:
break
logging(f'loading data {len(data)} from {path}')
return data, labels
with open(path, 'r', encoding='utf-8') as file:
for line in file:
line = line.strip('\n')
data.append(line[:-1])
labels.append(int(line[-1]))
logging(f'loading data {len(data)} from {path}')
return data, labels
def __init__(self,
origin_data_tokens,
word_dim: int = 100,
vocab_limit_size=80000,
is_using_pretrained=True,
word_vec_file_path=r'./static/glove.6B.100d.txt'):
self.file_path = word_vec_file_path
self.word_dim = word_dim
self.word_dict = {}
self.word_count = {}
self.vectors = None
self.num = 0
self.data_tokens = []
self.words_vocab = []
assert len(origin_data_tokens) > 0
self.data_tokens = origin_data_tokens
self.__build_words_index()
self.__limit_dict_size(vocab_limit_size)
if is_using_pretrained:
logging(f'building word vectors from {self.file_path}')
self.__read_pretrained_word_vecs()
logging(f'word vectors has been built! dict size is {self.num}')
shuffle=True,
num_workers=4)
test_data = DataLoader(test_dataset_orig,
batch_size=AttackConfig.batch_size,
shuffle=False,
num_workers=4)
return train_data, test_data
def save_config(path):
copyfile(config_path, path + r'/config.txt')
if __name__ == '__main__':
if AttackConfig.train_multi_cuda:
logging('Using cuda device gpu: ' + str(AttackConfig.multi_cuda_idx))
else:
logging('Using cuda device gpu: ' + str(AttackConfig.cuda_idx))
cur_dir = AttackConfig.output_dir + '/seq2seq_model/' + AttackConfig.dataset + '/' + str(
int(time.time()))
# make output directory if it doesn't already exist
if not os.path.isdir(cur_dir):
os.makedirs(cur_dir)
logging('Saving into directory' + cur_dir)
save_config(cur_dir)
baseline_model_builder = BaselineModelBuilder(AttackConfig.dataset,
AttackConfig.baseline_model,
AttackConfig.train_device,
is_load=True)
def train_bert_baseline_Entailment(model, train_data, test_data,
criterion_baseline_model,
optimizer_baseline_model, cur_dir):
best_accuracy = 0.0
for epoch in range(BaselineConfig.epochs):
logging(f'epoch {epoch} start')
logging(f'epoch {epoch} train baseline_model')
model.train()
loss_mean = 0.0
for _, _, _, _, _, _, x, x_mask, x_type, label in train_data:
x, x_mask, x_type, label = x.to(
BaselineConfig.train_device), x_mask.to(
BaselineConfig.train_device), x_type.to(
BaselineConfig.train_device), label.to(
BaselineConfig.train_device)
logits = model(x, x_mask, x_type)
optimizer_baseline_model.zero_grad()
loss = criterion_baseline_model(logits, label)
loss_mean += loss.item()
loss.backward()
optimizer_baseline_model.step()
loss_mean /= len(train_data)
logging(f"epoch {epoch} train_loss is {loss_mean}")
eval_accuracy = eval_bert_baseline_Entailment(model, test_data)
logging(f"epoch {epoch} test_acc is {eval_accuracy}")
if best_accuracy < eval_accuracy:
best_accuracy = eval_accuracy
logging('Saveing baseline models...')
torch.save(model.state_dict(), cur_dir + r'/baseline_model.pt')
if loss_mean < 0.1:
logging(f'best accuracy is {best_accuracy}')
break
logging(f'best accuracy is {best_accuracy}')
def generate_adversarial_samples(self,
path,
adv_method: str,
verbose=False,
use_typos=True,
tokenizer=None,
vocab=None,
net=None,
change_log_path=None,
sub_rate_limit=None):
assert len(self.datas) > 0
self.adv_methods = {
'PWWS': self.get_fool_sentence_pwws,
'TEXTFOOL': self.get_fool_sentence_textfool,
'RANDOM': self.get_fool_sentence_random,
}
assert adv_method in self.adv_methods
self.adv_methods = self.adv_methods[adv_method]
self.use_typos = use_typos
self.verbose = verbose
self.adv_datas = []
self.tokenizer = tokenizer
self.vocab = vocab
self.net = net
self.log_path = change_log_path
self.sub_rate_limit = sub_rate_limit
success_num = 0
failure_num = 0
try_all = 0
logging(
f'generate adversarial samples {len(self.datas)} with {adv_method} to {path}'
)
# assert net.training == False
for idx, data in enumerate(self.datas):
adv_s, flag, end = self.adv_methods(data, self.labels[idx], idx)
self.adv_datas.append(str(adv_s))
if flag == 1:
success_num += 1
try_all += 1
logging(
f'The {idx}th adv successfully crafted, '
f'success rate is {success_num/try_all:.5f}, cost {end:.2f} seconds'
)
elif flag == 0:
failure_num += 1
try_all += 1
logging(
f'The {idx}th adv example failed crafted, '
f'fail rate is {failure_num/try_all:.5f}, cost {end:.2f} seconds'
)
del adv_s
del flag
del end
if (idx + 1) % 100 == 0:
write_standard_data(self.adv_datas,
self.labels[idx - 99:idx + 1], path, 'a')
self.adv_datas.clear()
gc.collect()
if args.verbose:
now = idx + 1
acc = 1 - success_num / now
success_rate = success_num / try_all if try_all > 0 else 0.0
failure_rate = 1 - success_rate
vis.line(X=[now],
Y=[acc],
env=env_name,
win='rate',
name='acc',
update='append')
vis.line(X=[now],
Y=[success_rate],
env=env_name,
win='rate',
name='att_success_rate',
update='append')
vis.line(X=[now],
Y=[failure_rate],
env=env_name,
win='rate',
name='att_failure_rate',
update='append')
logging(f'try to generate adv_samples {try_all} '
f'generate successfully {success_num} '
f'failed {failure_num} '
f'origin samples num is {len(self.datas)}')
if len(self.adv_datas) > 0:
write_standard_data(self.adv_datas,
self.labels[len(self.adv_datas):], path, 'a')
def data2token(self):
logging(f'data is train {self.is_train} is to tokens!')
assert self.data is not None
for sen in self.data:
self.data_token.append(self.tokenizer(sen))
else:
if BaselineConfig.baseline_model == 'Bert':
train_bert_baseline_Classification(model, train_data, test_data,
criterion_baseline_model,
optimizer_baseline_model,
cur_dir)
else:
train_baseline_Classification(model, train_data, test_data,
criterion_baseline_model,
optimizer_baseline_model, cur_dir)
if __name__ == "__main__":
logging('Using cuda device gpu: ' + str(BaselineConfig.cuda_idx))
cur_dir = BaselineConfig.output_dir + '/baseline_model/' + BaselineConfig.dataset + '/' + BaselineConfig.baseline_model + '/' + str(
int(time.time()))
# make output directory if it doesn't already exist
if not os.path.isdir(cur_dir):
os.makedirs(cur_dir)
logging('Saving into directory' + cur_dir)
save_config(cur_dir)
train_data, test_data = build_dataset()
model = build_model().to(BaselineConfig.train_device)
logging('Training Baseline Model...')
criterion_baseline_model = nn.CrossEntropyLoss().to(
BaselineConfig.train_device)
optimizer_baseline_model = optim.Adam(
def main(epochs: int, learning_rate: float):
global best_acc
global best_path
global note
global best_state
criterion = nn.CrossEntropyLoss().to(config_device)
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
# optimizer = optim.SGD(net.parameters(), lr=learning_rate)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.95,
patience=3,
verbose=True,
min_lr=3e-9)
warmup_scheduler = optim.lr_scheduler.LambdaLR(optimizer,
lr_lambda=lambda ep: 1e-2
if ep < 3 else 1)
loss = 0.0
train_loss = 0.0
global_batch_idx = 1
acc_all = []
test_best_loss = 1e5
def evaluate(now_ep) -> float:
test_loss = 0.0
logging('starting evaluate!')
net.eval()
with torch.no_grad():
correct = 0
total = 0
for (X, label) in tqdm(test_data):
X, label = X.to(config_device), label.to(config_device)
logits = net(X)
test_loss += criterion(logits, label).item()
predict = logits.argmax(dim=1)
correct += predict.eq(label).float().sum().item()
total += X.size(0)
correct /= total
test_loss /= len(test_data)
logging(
f'epoch {now_ep} evaluate done! test acc {correct:.5f}, best acc{best_acc:.5f}, test batch loss {test_loss:.5f}'
)
return correct, test_loss
for ep in range(epochs):
logging(f'epoch {ep} start!')
net.train()
train_loss = 0.0
for batch_idx, (X, label) in enumerate(tqdm(train_data)):
X, label = X.to(config_device), label.to(config_device)
logits = net(X)
loss = criterion(logits, label)
train_loss += loss.item()
global_batch_idx += 1
optimizer.zero_grad()
loss.backward()
optimizer.step()
if global_batch_idx % 2000 == 0:
logging(
f'global batch {global_batch_idx}, loss is {train_loss/(batch_idx+1):.5f}'
)
train_loss /= len(train_data)
logging(
f'epoch {ep} train done! averge train loss is {train_loss:.5f}')
ep_test_acc, test_loss = evaluate(ep)
acc_all.append(ep_test_acc)
if ep < 4: warmup_scheduler.step(ep)
else: scheduler.step(test_loss, epoch=ep)
if args.verbose:
vis.line(X=[ep + 1],
Y=[train_loss],
env=env_name,
win='loss',
name='train_loss',
update='append')
vis.line(X=[ep + 1],
Y=[test_loss],
env=env_name,
win='loss',
name='test_loss',
update='append')
vis.line(X=[ep + 1],
Y=[ep_test_acc],
env=env_name,
win='acc',
name='test_acc',
update='append')
if ep_test_acc > best_acc or (ep_test_acc == best_acc
and test_loss < test_best_loss):
best_acc = ep_test_acc
test_best_loss = test_loss
best_path = config_model_save_path[dataset_name].format(
net.model_name, best_acc, get_time(), note)
best_state = copy.deepcopy(net.state_dict())
if (ep + 1) % (epochs // 3) == 0 and best_state:
logging(f'saving model in {best_path} best acc {best_acc:.5f}')
torch.save(best_state, best_path)
best_state = None
count = epochs - epochs // 2
acc_all = sum(acc_all[epochs // 2:])
acc_all /= count
if best_state is not None: torch.save(best_state, best_path)
logging(
f'train {epochs} done! The last train loss is {train_loss/len(test_data):.5f}!'
)
logging(
f'The last {count} epoch test average acc is {acc_all:.5f}, best acc is {best_acc:.5f}\n'
f'best model saved in {best_path}')
net = build_LSTM_model(dataset_name,
vocab,
config_device,
is_bid=True,
syn=syn,
is_load=is_load_model,
is_adv=args.adv)
# net = nn.DataParallel(net, device_ids=[0, 1])
net.to(config_device)
best_path = config_model_load_path[dataset_name].get(net.model_name)
best_state = None
best_acc = 0.0 if is_load_model == False else float(
re.findall("_\d.\d+_", best_path)[0][1:-1])
if is_load_model:
logging(f'loading net model from {best_path}, acc is {best_acc}')
else:
pass # net.apply(weights_init)
def main(epochs: int, learning_rate: float):
global best_acc
global best_path
global note
global best_state
criterion = nn.CrossEntropyLoss().to(config_device)
optimizer = optim.Adam(net.parameters(), lr=learning_rate)
# optimizer = optim.SGD(net.parameters(), lr=learning_rate)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
with torch.no_grad():
model.eval()
acc_sum = 0
n = 0
for _, x_mask, x, label in test_data:
x, x_mask, label = x.to(BaselineConfig.train_device), x_mask.to(
BaselineConfig.train_device), label.to(
BaselineConfig.train_device)
logits = model(x, x_mask)
acc_sum += (logits.argmax(dim=1) == label).float().sum().item()
n += label.shape[0]
return acc_sum / n
if __name__ == "__main__":
baseline_model_builder = BaselineModelBuilder('AGNEWS',
'LSTM',
BaselineConfig.train_device,
is_load=True)
test_dataset_orig = AGNEWS_Dataset(
train_data=False,
attack_vocab=baseline_model_builder.vocab,
debug_mode=False)
test_data = DataLoader(test_dataset_orig,
batch_size=BaselineConfig.batch_size,
shuffle=False,
num_workers=4)
logging(
eval_bert_baseline_Classification(baseline_model_builder.net,
test_data))
test_dataset_orig = SST2_Dataset(train_data=False,
attack_vocab=attack_vocab,
debug_mode=AttackConfig.debug_mode)
train_data = DataLoader(train_dataset_orig,
batch_size=AttackConfig.batch_size,
shuffle=True,
num_workers=4)
test_data = DataLoader(test_dataset_orig,
batch_size=AttackConfig.batch_size,
shuffle=False,
num_workers=4)
return train_data, test_data
if __name__ == '__main__':
logging('Using cuda device gpu: ' + str(AttackConfig.cuda_idx))
cur_dir = AttackConfig.output_dir + '/gan_model/' + AttackConfig.dataset + '/' + AttackConfig.baseline_model + '/' + str(
int(time.time()))
cur_dir_models = cur_dir + '/models'
# make output directory if it doesn't already exist
if not os.path.isdir(cur_dir):
os.makedirs(cur_dir)
os.makedirs(cur_dir_models)
logging('Saving into directory' + cur_dir)
save_config(cur_dir)
baseline_model_builder = BaselineModelBuilder(AttackConfig.dataset,
AttackConfig.baseline_model,
AttackConfig.train_device,
is_load=True)
def evaluate_gan(test_data, Seq2Seq_model, gan_gen, gan_adv, dir,
attack_vocab):
Seq2Seq_model.eval()
gan_gen.eval()
gan_adv.eval()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
logging(f'Saving evaluate of gan outputs into {dir}')
with torch.no_grad():
for x, x_mask, y, _ in test_data:
x, x_mask, y = x.to(AttackConfig.train_device), x_mask.to(
AttackConfig.train_device), y.to(AttackConfig.train_device)
# sentence -> encoder -> decoder
Seq2Seq_outputs = Seq2Seq_model(x, x_mask, is_noise=False)
# Seq2Seq_idx: [batch, seq_len]
Seq2Seq_idx = Seq2Seq_outputs.argmax(dim=2)
# sentence -> encoder -> adversary -> generator -> decoder
# eagd_outputs: [batch, seq_len, vocab_size]
eagd_outputs = Seq2Seq_model(x,
x_mask,
is_noise=False,
generator=gan_gen,
adversary=gan_adv)
# eagd_idx: [batch_size, sen_len]
eagd_idx = eagd_outputs.argmax(dim=2)
if attack_vocab:
with open(dir, 'a') as f:
for i in range(len(y)):
f.write('------orginal sentence---------\n')
f.write(' '.join(
[attack_vocab.get_word(token)
for token in y[i]]) + '\n')
f.write('------setence -> encoder -> decoder-------\n')
f.write(' '.join([
attack_vocab.get_word(token)
for token in Seq2Seq_idx[i]
]) + '\n')
f.write(
'------sentence -> encoder -> inverter -> generator -> decoder-------\n'
)
f.write(' '.join([
attack_vocab.get_word(token)
for token in eagd_idx[i]
]) + '\n' * 2)
else:
with open(dir, 'a') as f:
for i in range(len(y)):
f.write('------orginal sentence---------\n')
f.write(
' '.join(tokenizer.convert_ids_to_tokens(y[i])) +
'\n')
f.write('------setence -> encoder -> decoder-------\n')
f.write(' '.join(
tokenizer.convert_ids_to_tokens(Seq2Seq_idx[i])) +
'\n')
f.write(
'------sentence -> encoder -> inverter -> generator -> decoder-------\n'
)
f.write(' '.join(
tokenizer.convert_ids_to_tokens(eagd_idx[i])) +
'\n' * 2)
def save_all_models(Seq2Seq_model, gan_gen, gan_adv, dir):
logging('Saving models...')
torch.save(Seq2Seq_model.state_dict(), dir + '/Seq2Seq_model.pt')
torch.save(gan_gen.state_dict(), dir + '/gan_gen.pt')
torch.save(gan_adv.state_dict(), dir + '/gan_adv.pt')
net = build_LSTM_model(dataset_name,
vocab0,
config_device,
is_bid=True,
syn=None)
elif model == 'BidLSTM_adv':
net = build_LSTM_model(dataset_name,
vocab1,
config_device,
is_bid=True,
syn=None,
is_adv=True)
elif model == 'BidLSTM_enhanced':
net = build_LSTM_model(dataset_name,
vocab0,
config_device,
is_bid=True,
syn=syn)
logging('evaluate origin test data')
test_acc = evaluate(test_data, net)
logging('evaluate origin 1k clean data')
clean1k_acc = evaluate(clean_data, net)
logging('evaluate adversarial 1k data')
adv1k_acc = evaluate(adv_data, net)
results.append((test_acc, clean1k_acc, adv1k_acc))
logs.append(read_fool_log(adv_log_path))
write_results_to_file(models, results, logs)
