def eval_iters(ae_model, dis_model): eval_data_loader = non_pair_data_loader( batch_size=1, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk, max_sequence_length=args.max_sequence_length, vocab_size=args.vocab_size) eval_file_list = [ args.data_path + 'sentiment.test.0', args.data_path + 'sentiment.test.1', ] eval_label_list = [ [0], [1], ] eval_data_loader.create_batches(eval_file_list, eval_label_list, if_shuffle=False) gold_ans = load_human_answer(args.data_path) assert len(gold_ans) == eval_data_loader.num_batch add_log("Start eval process.") ae_model.eval() dis_model.eval() for it in range(eval_data_loader.num_batch): batch_sentences, tensor_labels, \ tensor_src, tensor_src_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, tensor_ntokens = eval_data_loader.next_batch() print("------------%d------------" % it) print(id2text_sentence(tensor_tgt_y[0], args.id_to_word)) print("origin_labels", tensor_labels) latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_tgt_mask) generator_text = ae_model.greedy_decode( latent, max_len=args.max_sequence_length, start_id=args.id_bos) print(id2text_sentence(generator_text[0], args.id_to_word)) # Define target label target = get_cuda(torch.tensor([[1.0]], dtype=torch.float)) if tensor_labels[0].item() > 0.5: target = get_cuda(torch.tensor([[0.0]], dtype=torch.float)) print("target_labels", target) modify_text = fgim_attack(dis_model, latent, target, ae_model, args.max_sequence_length, args.id_bos, id2text_sentence, args.id_to_word, gold_ans[it]) add_output(modify_text) output_text = str(it) + ":\ngold: " + id2text_sentence( gold_ans[it], args.id_to_word) + "\nmodified: " + modify_text add_output(output_text) add_result( str(it) + ":\n" + str( calc_bleu(id2text_sentence(gold_ans[it], args.id_to_word), modify_text))) return
def plot_tsne(ae_model, dis_model, epsilon=2, step=0): eval_data_loader = non_pair_data_loader( batch_size=500, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk, max_sequence_length=args.max_sequence_length, vocab_size=args.vocab_size) eval_file_list = [ args.data_path + 'sentiment.test.0', args.data_path + 'sentiment.test.1', ] eval_label_list = [ [0], [1], ] eval_data_loader.create_batches(eval_file_list, eval_label_list, if_shuffle=False) gold_ans = load_human_answer(args.data_path) ae_model.eval() dis_model.eval() latents, labels = [], [] it = 0 for _ in range(eval_data_loader.num_batch): batch_sentences, tensor_labels, \ tensor_src, tensor_src_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, tensor_ntokens = eval_data_loader.next_batch() print("------------%d------------" % it) print(id2text_sentence(tensor_tgt_y[0], args.id_to_word)) print("origin_labels", tensor_labels[0].item()) latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_tgt_mask) # Define target label target = get_cuda( torch.ones((tensor_labels.size(0), 1), dtype=torch.float)) target = target - tensor_labels if step > 0: latent, modified_text = fgim_step(dis_model, latent, target, ae_model, args.max_sequence_length, args.id_bos, id2text_sentence, args.id_to_word, gold_ans[it], epsilon, step) latents.append(latent) labels.append(tensor_labels) it += tensor_labels.size(0) latents = torch.cat(latents, dim=0).detach().cpu().numpy() labels = torch.cat(labels, dim=0).squeeze().detach().cpu().numpy() tsne_plot_representation(latents, labels, f"tsne_step{step}_eps{epsilon}")
def plot_tsne(ae_model, dis_model): epsilon = 2 step = 1 eval_data_loader = non_pair_data_loader( batch_size=1, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk, max_sequence_length=args.max_sequence_length, vocab_size=args.vocab_size) eval_file_list = [ args.data_path + 'sentiment.test.0', args.data_path + 'sentiment.test.1', ] eval_label_list = [ [0], [1], ] eval_data_loader.create_batches(eval_file_list, eval_label_list, if_shuffle=False) gold_ans = load_human_answer(args.data_path) assert len(gold_ans) == eval_data_loader.num_batch ae_model.eval() dis_model.eval() latents, labels = [], [] for it in range(eval_data_loader.num_batch): batch_sentences, tensor_labels, \ tensor_src, tensor_src_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, tensor_ntokens = eval_data_loader.next_batch() print("------------%d------------" % it) print(id2text_sentence(tensor_tgt_y[0], args.id_to_word)) print("origin_labels", tensor_labels.item()) latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_tgt_mask) # Define target label target = get_cuda(torch.tensor([[1.0]], dtype=torch.float)) if tensor_labels[0].item() > 0.5: target = get_cuda(torch.tensor([[0.0]], dtype=torch.float)) modified_latent, modified_text = fgim_step( dis_model, latent, target, ae_model, args.max_sequence_length, args.id_bos, id2text_sentence, args.id_to_word, gold_ans[it], epsilon, step) latents.append(modified_latent) labels.append(tensor_labels.item()) latents = torch.cat(latents, dim=0).detach().cpu().numpy() labels = numpy.array(labels) tsne_plot_representation(latents, labels)
def sedat_eval(args, ae_model, f, deb): """ Input: Original latent representation z : (n_batch, batch_size, seq_length, latent_size) Output: An optimal modified latent representation z' """ max_sequence_length = args.max_sequence_length id_bos = args.id_bos id_to_word = args.id_to_word limit_batches = args.limit_batches eval_data_loader = non_pair_data_loader( batch_size=args.batch_size, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk, max_sequence_length=args.max_sequence_length, vocab_size=args.vocab_size) file_list = [args.test_data_file] eval_data_loader.create_batches(args, file_list, if_shuffle=False, n_samples=args.test_n_samples) if args.references_files: gold_ans = load_human_answer(args.references_files, args.text_column) assert len(gold_ans) == eval_data_loader.num_batch else: gold_ans = None add_log(args, "Start eval process.") ae_model.eval() f.eval() deb.eval() text_z_prime = {} text_z_prime = { "source": [], "origin_labels": [], "before": [], "after": [], "change": [], "pred_label": [] } if gold_ans is not None: text_z_prime["gold_ans"] = [] z_prime = [] n_batches = 0 for it in tqdm.tqdm(list(range(eval_data_loader.num_batch)), desc="SEDAT"): _, tensor_labels, \ tensor_src, tensor_src_mask, tensor_src_attn_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, _ = eval_data_loader.next_batch() # only on negative example negative_examples = ~(tensor_labels.squeeze() == args.positive_label) tensor_labels = tensor_labels[negative_examples].squeeze( 0) # .view(1, -1) tensor_src = tensor_src[negative_examples].squeeze(0) tensor_src_mask = tensor_src_mask[negative_examples].squeeze(0) tensor_src_attn_mask = tensor_src_attn_mask[negative_examples].squeeze( 0) tensor_tgt_y = tensor_tgt_y[negative_examples].squeeze(0) tensor_tgt = tensor_tgt[negative_examples].squeeze(0) tensor_tgt_mask = tensor_tgt_mask[negative_examples].squeeze(0) if negative_examples.any(): if gold_ans is not None: text_z_prime["gold_ans"].append(gold_ans[it]) text_z_prime["source"].append( [id2text_sentence(t, args.id_to_word) for t in tensor_tgt_y]) text_z_prime["origin_labels"].append(tensor_labels.cpu().numpy()) origin_data, _ = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_src_attn_mask, tensor_tgt_mask) generator_id = ae_model.greedy_decode(origin_data, max_len=max_sequence_length, start_id=id_bos) generator_text = [ id2text_sentence(gid, id_to_word) for gid in generator_id ] text_z_prime["before"].append(generator_text) data = deb(origin_data, mask=None) data = torch.sum(ae_model.sigmoid(data), dim=1) # (batch_size, d_model) #logit = ae_model.decode(data.unsqueeze(1), tensor_tgt, tensor_tgt_mask) # (batch_size, max_tgt_seq, d_model) #output = ae_model.generator(logit) # (batch_size, max_seq, vocab_size) y_hat = f.forward(data) y_hat = y_hat.round().int() z_prime.append(data) generator_id = ae_model.greedy_decode(data, max_len=max_sequence_length, start_id=id_bos) generator_text = [ id2text_sentence(gid, id_to_word) for gid in generator_id ] text_z_prime["after"].append(generator_text) text_z_prime["change"].append([True] * len(y_hat)) text_z_prime["pred_label"].append([y_.item() for y_ in y_hat]) n_batches += 1 if n_batches > limit_batches: break write_text_z_in_file(args, text_z_prime) add_log(args, "") add_log(args, "Saving model modify embedding %s ..." % args.current_save_path) torch.save(z_prime, os.path.join(args.current_save_path, 'z_prime_sedat.pkl')) return z_prime, text_z_prime
def fgim_algorithm(args, ae_model, dis_model): batch_size = 1 test_data_loader = non_pair_data_loader( batch_size=batch_size, id_bos=args.id_bos, id_eos=args.id_eos, id_unk=args.id_unk, max_sequence_length=args.max_sequence_length, vocab_size=args.vocab_size) file_list = [args.test_data_file] test_data_loader.create_batches(args, file_list, if_shuffle=False, n_samples=args.test_n_samples) if args.references_files: gold_ans = load_human_answer(args.references_files, args.text_column) assert len(gold_ans) == test_data_loader.num_batch else: gold_ans = [[None] * batch_size] * test_data_loader.num_batch add_log(args, "Start eval process.") ae_model.eval() dis_model.eval() fgim_our = True if fgim_our: # for FGIM z_prime, text_z_prime = fgim(test_data_loader, args, ae_model, dis_model, gold_ans=gold_ans) write_text_z_in_file(args, text_z_prime) add_log( args, "Saving model modify embedding %s ..." % args.current_save_path) torch.save(z_prime, os.path.join(args.current_save_path, 'z_prime_fgim.pkl')) else: for it in range(test_data_loader.num_batch): batch_sentences, tensor_labels, \ tensor_src, tensor_src_mask, tensor_src_attn_mask, tensor_tgt, tensor_tgt_y, \ tensor_tgt_mask, tensor_ntokens = test_data_loader.next_batch() print("------------%d------------" % it) print(id2text_sentence(tensor_tgt_y[0], args.id_to_word)) print("origin_labels", tensor_labels) latent, out = ae_model.forward(tensor_src, tensor_tgt, tensor_src_mask, tensor_src_attn_mask, tensor_tgt_mask) generator_text = ae_model.greedy_decode( latent, max_len=args.max_sequence_length, start_id=args.id_bos) print(id2text_sentence(generator_text[0], args.id_to_word)) # Define target label target = get_cuda(torch.tensor([[1.0]], dtype=torch.float), args) if tensor_labels[0].item() > 0.5: target = get_cuda(torch.tensor([[0.0]], dtype=torch.float), args) add_log(args, "target_labels : %s" % target) modify_text = fgim_attack(dis_model, latent, target, ae_model, args.max_sequence_length, args.id_bos, id2text_sentence, args.id_to_word, gold_ans[it]) add_output(args, modify_text)