def eval_iters(ae_model, dis_model):
eval_data_loader = non_pair_data_loader(
batch_size=1,
id_bos=args.id_bos,
id_eos=args.id_eos,
id_unk=args.id_unk,
max_sequence_length=args.max_sequence_length,
vocab_size=args.vocab_size)
eval_file_list = [
args.data_path + 'sentiment.test.0',
args.data_path + 'sentiment.test.1',
]
eval_label_list = [
[0],
[1],
]
eval_data_loader.create_batches(eval_file_list,
eval_label_list,
if_shuffle=False)
gold_ans = load_human_answer(args.data_path)
assert len(gold_ans) == eval_data_loader.num_batch
add_log("Start eval process.")
ae_model.eval()
dis_model.eval()
for it in range(eval_data_loader.num_batch):
batch_sentences, tensor_labels, \
tensor_src, tensor_src_mask, tensor_tgt, tensor_tgt_y, \
tensor_tgt_mask, tensor_ntokens = eval_data_loader.next_batch()
print("------------%d------------" % it)
print(id2text_sentence(tensor_tgt_y[0], args.id_to_word))
print("origin_labels", tensor_labels)
latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask,
tensor_tgt_mask)
generator_text = ae_model.greedy_decode(
latent, max_len=args.max_sequence_length, start_id=args.id_bos)
print(id2text_sentence(generator_text[0], args.id_to_word))
# Define target label
target = get_cuda(torch.tensor([[1.0]], dtype=torch.float))
if tensor_labels[0].item() > 0.5:
target = get_cuda(torch.tensor([[0.0]], dtype=torch.float))
print("target_labels", target)
modify_text = fgim_attack(dis_model, latent, target, ae_model,
args.max_sequence_length, args.id_bos,
id2text_sentence, args.id_to_word,
gold_ans[it])
add_output(modify_text)
output_text = str(it) + ":\ngold: " + id2text_sentence(
gold_ans[it], args.id_to_word) + "\nmodified: " + modify_text
add_output(output_text)
add_result(
str(it) + ":\n" + str(
calc_bleu(id2text_sentence(gold_ans[it], args.id_to_word),
modify_text)))
return
def plot_tsne(ae_model, dis_model, epsilon=2, step=0):
eval_data_loader = non_pair_data_loader(
batch_size=500,
id_bos=args.id_bos,
id_eos=args.id_eos,
id_unk=args.id_unk,
max_sequence_length=args.max_sequence_length,
vocab_size=args.vocab_size)
eval_file_list = [
args.data_path + 'sentiment.test.0',
args.data_path + 'sentiment.test.1',
]
eval_label_list = [
[0],
[1],
]
eval_data_loader.create_batches(eval_file_list,
eval_label_list,
if_shuffle=False)
gold_ans = load_human_answer(args.data_path)
ae_model.eval()
dis_model.eval()
latents, labels = [], []
it = 0
for _ in range(eval_data_loader.num_batch):
batch_sentences, tensor_labels, \
tensor_src, tensor_src_mask, tensor_tgt, tensor_tgt_y, \
tensor_tgt_mask, tensor_ntokens = eval_data_loader.next_batch()
print("------------%d------------" % it)
print(id2text_sentence(tensor_tgt_y[0], args.id_to_word))
print("origin_labels", tensor_labels[0].item())
latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask,
tensor_tgt_mask)
# Define target label
target = get_cuda(
torch.ones((tensor_labels.size(0), 1), dtype=torch.float))
target = target - tensor_labels
if step > 0:
latent, modified_text = fgim_step(dis_model, latent, target,
ae_model,
args.max_sequence_length,
args.id_bos, id2text_sentence,
args.id_to_word, gold_ans[it],
epsilon, step)
latents.append(latent)
labels.append(tensor_labels)
it += tensor_labels.size(0)
latents = torch.cat(latents, dim=0).detach().cpu().numpy()
labels = torch.cat(labels, dim=0).squeeze().detach().cpu().numpy()
tsne_plot_representation(latents, labels, f"tsne_step{step}_eps{epsilon}")
def plot_tsne(ae_model, dis_model):
epsilon = 2
step = 1
eval_data_loader = non_pair_data_loader(
batch_size=1,
id_bos=args.id_bos,
id_eos=args.id_eos,
id_unk=args.id_unk,
max_sequence_length=args.max_sequence_length,
vocab_size=args.vocab_size)
eval_file_list = [
args.data_path + 'sentiment.test.0',
args.data_path + 'sentiment.test.1',
]
eval_label_list = [
[0],
[1],
]
eval_data_loader.create_batches(eval_file_list,
eval_label_list,
if_shuffle=False)
gold_ans = load_human_answer(args.data_path)
assert len(gold_ans) == eval_data_loader.num_batch
ae_model.eval()
dis_model.eval()
latents, labels = [], []
for it in range(eval_data_loader.num_batch):
batch_sentences, tensor_labels, \
tensor_src, tensor_src_mask, tensor_tgt, tensor_tgt_y, \
tensor_tgt_mask, tensor_ntokens = eval_data_loader.next_batch()
print("------------%d------------" % it)
print(id2text_sentence(tensor_tgt_y[0], args.id_to_word))
print("origin_labels", tensor_labels.item())
latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask,
tensor_tgt_mask)
# Define target label
target = get_cuda(torch.tensor([[1.0]], dtype=torch.float))
if tensor_labels[0].item() > 0.5:
target = get_cuda(torch.tensor([[0.0]], dtype=torch.float))
modified_latent, modified_text = fgim_step(
dis_model, latent, target, ae_model, args.max_sequence_length,
args.id_bos, id2text_sentence, args.id_to_word, gold_ans[it],
epsilon, step)
latents.append(modified_latent)
labels.append(tensor_labels.item())
latents = torch.cat(latents, dim=0).detach().cpu().numpy()
labels = numpy.array(labels)
tsne_plot_representation(latents, labels)
def sedat_eval(args, ae_model, f, deb):
"""
Input:
Original latent representation z : (n_batch, batch_size, seq_length, latent_size)
Output:
An optimal modified latent representation z'
"""
max_sequence_length = args.max_sequence_length
id_bos = args.id_bos
id_to_word = args.id_to_word
limit_batches = args.limit_batches
eval_data_loader = non_pair_data_loader(
batch_size=args.batch_size,
id_bos=args.id_bos,
id_eos=args.id_eos,
id_unk=args.id_unk,
max_sequence_length=args.max_sequence_length,
vocab_size=args.vocab_size)
file_list = [args.test_data_file]
eval_data_loader.create_batches(args,
file_list,
if_shuffle=False,
n_samples=args.test_n_samples)
if args.references_files:
gold_ans = load_human_answer(args.references_files, args.text_column)
assert len(gold_ans) == eval_data_loader.num_batch
else:
gold_ans = None
add_log(args, "Start eval process.")
ae_model.eval()
f.eval()
deb.eval()
text_z_prime = {}
text_z_prime = {
"source": [],
"origin_labels": [],
"before": [],
"after": [],
"change": [],
"pred_label": []
}
if gold_ans is not None:
text_z_prime["gold_ans"] = []
z_prime = []
n_batches = 0
for it in tqdm.tqdm(list(range(eval_data_loader.num_batch)), desc="SEDAT"):
_, tensor_labels, \
tensor_src, tensor_src_mask, tensor_src_attn_mask, tensor_tgt, tensor_tgt_y, \
tensor_tgt_mask, _ = eval_data_loader.next_batch()
# only on negative example
negative_examples = ~(tensor_labels.squeeze() == args.positive_label)
tensor_labels = tensor_labels[negative_examples].squeeze(
0) # .view(1, -1)
tensor_src = tensor_src[negative_examples].squeeze(0)
tensor_src_mask = tensor_src_mask[negative_examples].squeeze(0)
tensor_src_attn_mask = tensor_src_attn_mask[negative_examples].squeeze(
0)
tensor_tgt_y = tensor_tgt_y[negative_examples].squeeze(0)
tensor_tgt = tensor_tgt[negative_examples].squeeze(0)
tensor_tgt_mask = tensor_tgt_mask[negative_examples].squeeze(0)
if negative_examples.any():
if gold_ans is not None:
text_z_prime["gold_ans"].append(gold_ans[it])
text_z_prime["source"].append(
[id2text_sentence(t, args.id_to_word) for t in tensor_tgt_y])
text_z_prime["origin_labels"].append(tensor_labels.cpu().numpy())
origin_data, _ = ae_model.forward(tensor_src, tensor_tgt,
tensor_src_mask,
tensor_src_attn_mask,
tensor_tgt_mask)
generator_id = ae_model.greedy_decode(origin_data,
max_len=max_sequence_length,
start_id=id_bos)
generator_text = [
id2text_sentence(gid, id_to_word) for gid in generator_id
]
text_z_prime["before"].append(generator_text)
data = deb(origin_data, mask=None)
data = torch.sum(ae_model.sigmoid(data),
dim=1) # (batch_size, d_model)
#logit = ae_model.decode(data.unsqueeze(1), tensor_tgt, tensor_tgt_mask) # (batch_size, max_tgt_seq, d_model)
#output = ae_model.generator(logit) # (batch_size, max_seq, vocab_size)
y_hat = f.forward(data)
y_hat = y_hat.round().int()
z_prime.append(data)
generator_id = ae_model.greedy_decode(data,
max_len=max_sequence_length,
start_id=id_bos)
generator_text = [
id2text_sentence(gid, id_to_word) for gid in generator_id
]
text_z_prime["after"].append(generator_text)
text_z_prime["change"].append([True] * len(y_hat))
text_z_prime["pred_label"].append([y_.item() for y_ in y_hat])
n_batches += 1
if n_batches > limit_batches:
break
write_text_z_in_file(args, text_z_prime)
add_log(args, "")
add_log(args,
"Saving model modify embedding %s ..." % args.current_save_path)
torch.save(z_prime,
os.path.join(args.current_save_path, 'z_prime_sedat.pkl'))
return z_prime, text_z_prime
def fgim_algorithm(args, ae_model, dis_model):
batch_size = 1
test_data_loader = non_pair_data_loader(
batch_size=batch_size,
id_bos=args.id_bos,
id_eos=args.id_eos,
id_unk=args.id_unk,
max_sequence_length=args.max_sequence_length,
vocab_size=args.vocab_size)
file_list = [args.test_data_file]
test_data_loader.create_batches(args,
file_list,
if_shuffle=False,
n_samples=args.test_n_samples)
if args.references_files:
gold_ans = load_human_answer(args.references_files, args.text_column)
assert len(gold_ans) == test_data_loader.num_batch
else:
gold_ans = [[None] * batch_size] * test_data_loader.num_batch
add_log(args, "Start eval process.")
ae_model.eval()
dis_model.eval()
fgim_our = True
if fgim_our:
# for FGIM
z_prime, text_z_prime = fgim(test_data_loader,
args,
ae_model,
dis_model,
gold_ans=gold_ans)
write_text_z_in_file(args, text_z_prime)
add_log(
args,
"Saving model modify embedding %s ..." % args.current_save_path)
torch.save(z_prime,
os.path.join(args.current_save_path, 'z_prime_fgim.pkl'))
else:
for it in range(test_data_loader.num_batch):
batch_sentences, tensor_labels, \
tensor_src, tensor_src_mask, tensor_src_attn_mask, tensor_tgt, tensor_tgt_y, \
tensor_tgt_mask, tensor_ntokens = test_data_loader.next_batch()
print("------------%d------------" % it)
print(id2text_sentence(tensor_tgt_y[0], args.id_to_word))
print("origin_labels", tensor_labels)
latent, out = ae_model.forward(tensor_src, tensor_tgt,
tensor_src_mask,
tensor_src_attn_mask,
tensor_tgt_mask)
generator_text = ae_model.greedy_decode(
latent, max_len=args.max_sequence_length, start_id=args.id_bos)
print(id2text_sentence(generator_text[0], args.id_to_word))
# Define target label
target = get_cuda(torch.tensor([[1.0]], dtype=torch.float), args)
if tensor_labels[0].item() > 0.5:
target = get_cuda(torch.tensor([[0.0]], dtype=torch.float),
args)
add_log(args, "target_labels : %s" % target)
modify_text = fgim_attack(dis_model, latent, target, ae_model,
args.max_sequence_length, args.id_bos,
id2text_sentence, args.id_to_word,
gold_ans[it])
add_output(args, modify_text)
