def eval_bs_batch(batch: Batch, model: Seq2Seq, vocab: Vocab, *, pack_seq=True, beam_size=4,
min_out_len=1, max_out_len=None, len_in_words=True, best_only=True,
details: bool = True) -> Tuple[Optional[List[Dict[str, float]]], Optional[str]]:
"""
:param batch: a test batch of a single example
:param model: a trained summarizer
:param vocab: vocabulary of the trained summarizer
:param pack_seq: currently has no effect as batch size is 1
:param beam_size: the beam size
:param min_out_len: required minimum output length
:param max_out_len: required maximum output length (if None, use the model's own value)
:param len_in_words: if True, count output length in words instead of tokens (i.e. do not count
punctuations)
:param best_only: if True, run ROUGE only on the best hypothesis instead of all `beam size` many
:param details: if True, also return a string containing the result of this document
:return: two-level score lookup (hypothesis index => ROUGE metric => value)
Test a trained summarizer on a document using the beam search decoder.
"""
assert len(batch.examples) == 1
with torch.no_grad():
input_tensor = batch.input_tensor.to(DEVICE) # (src_len, 1)
if not pack_seq:
input_lengths = None
else:
# use PAD
input_lengths = batch.input_lengths
mask = create_mask(input_lengths)
# 返回beam_size大小的hypo
hypotheses = model.beam_search(input_tensor, input_lengths,
batch.ext_vocab_size, beam_size, min_out_len=min_out_len,
max_out_len=max_out_len, len_in_words=len_in_words, mask=mask)
if best_only:
to_decode = [hypotheses[0].tokens]
else:
to_decode = [h.tokens for h in hypotheses]
decoded_batch = decode_batch_output(to_decode, vocab, batch.oov_dict)
if details:
file_content = "[System Summary]\n" + format_tokens(decoded_batch[0])
else:
file_content = None
if batch.examples[0].tgt is not None: # run ROUGE if gold standard summary exists
gold_summaries = [batch.examples[0].tgt for _ in range(len(decoded_batch))]
scores = rouge(gold_summaries, decoded_batch)
if details:
file_content += "\n\n\n[Reference Summary]\n" + format_tokens(batch.examples[0].tgt)
file_content += "\n\n\n[ROUGE Scores]\n" + format_rouge_scores(scores[0]) + "\n"
else:
scores = None
if details:
file_content += "\n\n\n[Source Text]\n" + format_tokens(batch.examples[0].src)
return scores, file_content
def eval_texts_folders(args):
files1 = os.listdir(args.ref_dir)
files2 = os.listdir(args.cand_dir)
assert (files1 == files2)
ref = []
cand = []
for fn in files1:
with open(os.path.join(args.ref_dir, fn),
'r',
encoding="utf-8",
errors='ignore') as f:
ref.append([line.strip() for line in f.readlines()])
with open(os.path.join(args.cand_dir, fn),
'r',
encoding="utf-8",
errors='ignore') as f:
cand.append([line.strip() for line in f.readlines()])
rouge = Rouge()
rouge_score = rouge.get_scores([" ".join(p) for p in cand],
[" ".join(g) for g in ref],
avg=True)
formatted_rouge_score = format_rouge_scores(rouge_score)
rouge_table = format_rouge_table(rouge_score)
similarity_score = calc_sbert_similarity(ref, cand)
sentence_overlap = calc_sentence_coverage(cand, ref)
print(formatted_rouge_score)
print("Similarity score(sbert): %.3f" % similarity_score)
print("Sentence overlap: %.3f" % sentence_overlap)
print(rouge_table + " & %.3f & %.3f" %
(similarity_score, sentence_overlap))
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
rouge_output_file = os.path.join(
args.output_dir,
"text_eval_results_{}_{}.txt".format(os.path.basename(args.cand_dir),
os.path.basename(args.ref_dir)))
with open(rouge_output_file, 'w', encoding="utf-8") as f:
f.write(formatted_rouge_score)
f.write("Sentence overlap: %.3f\n" % sentence_overlap)
f.write("Similarity score(sbert): %.3f\n" % similarity_score)
f.write(rouge_table + " & %.3f & %.3f \\\\" %
(similarity_score, sentence_overlap))
def cnndm_lead_n(args, logger):
tokenizer = AutoTokenizer.from_pretrained(args.model_name)
eval_dataset = CNNDMBlobNoTokens(prefix='test', data_path=args.data_dir)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset,
sampler=eval_sampler,
batch_size=args.batch_size,
collate_fn=lambda x: x,
num_workers=args.num_workers)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_dataset))
gold = []
pred = []
for batch in tqdm(eval_dataloader, desc="Evaluating"):
stories = [x[0] for x in batch]
summaries = [x[1] for x in batch]
for story, summary in zip(stories, summaries):
pred.append(story[:3])
gold.append(summary)
rouge = Rouge()
simple_rouge_score_greed = np.mean(
[simple_rouge(every7[i], gold[i]) for i in range(len(gold))]) * 100
rouge_score_greed = format_rouge_scores(
rouge.get_scores([" ".join(p) for p in every7],
[" ".join(g) for g in gold],
avg=True))
similarity_score_greed = calc_sbert_similarity(every7, gold)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
rouge_output_file = os.path.join(args.output_dir, "cnndm_every7.txt")
with open(rouge_output_file, 'w', encoding="utf-8") as f:
f.write("lead-3")
f.write(rouge_score_greed)
f.write("\nSimple rouge score: %f\n" % simple_rouge_score_greed)
f.write("Similarity score(sbert): %f\n" % similarity_score_greed)
def cnndm_test_full(args, model, logger):
model = AutoExtSummarizer(args)
model.to(args.device)
model.eval()
tokenizer = AutoTokenizer.from_pretrained(args.model_name)
train_dataset = CNNDMBlobNoTokens(prefix='test',
data_path=args.data_dir,
label_key=args.label_key)
train_sampler = SequentialSampler(train_dataset)
model_collate_fn = functools.partial(collate,
pad_token_id=tokenizer.pad_token_id)
train_dataloader = DataLoader(train_dataset,
sampler=train_sampler,
batch_size=1,
collate_fn=single_collate,
num_workers=args.num_workers)
logger.info("***** Running CNNDM evaluation *****")
logger.info(" Num examples = %d", len(train_dataset))
gold = []
pred = []
for batch in tqdm(train_dataloader, desc="Evaluating"): #, disable=True):
summary = batch[0][2]
story = batch[0][0]
blocks = create_labeled_blocks(args, batch[0], tokenizer)
block_scores = []
memory = None
for block in blocks:
_batch = Batch([block], pad_token_id=tokenizer.pad_token_id)
source = _batch.src.to(args.device)
encoder_mask = _batch.mask.to(args.device)
clss = _batch.clss.to(args.device)
cls_mask = _batch.mask_cls.to(args.device).bool()
with torch.no_grad():
sent_scores, mask, memory = model(source,
encoder_mask,
clss,
cls_mask,
memory=memory)
#Seperates padding from the ones that are actually 0
sent_scores = sent_scores + mask.float()
sent_scores = sent_scores.cpu().data.numpy()
block_scores.extend(sent_scores[0])
selected_ids = np.argsort(block_scores)[::-1]
_pred = []
for i in selected_ids:
candidate = story[i].strip()
if (not _block_tri(candidate, _pred)):
_pred.append(candidate)
if len(_pred) == 3:
break
pred.append(_pred)
gold.append(summary)
#python rouge implementation
rouge = Rouge()
rouge_score = rouge.get_scores([" ".join(p) for p in pred],
[" ".join(g) for g in gold],
avg=True)
rouge_score_formatted = format_rouge_scores(rouge_score)
rouge_table = format_rouge_table(rouge_score)
similarity_score = calc_sbert_similarity(pred, gold)
#similarity_score = 0
print(rouge_score_formatted)
print("Similarity score(sbert): %.3f" % similarity_score)
print(rouge_table + " & %.3f" % similarity_score)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
rouge_output_file = os.path.join(
args.output_dir, "cnndm_test_full_results_{}_{}.txt".format(
args.model_name,
os.path.basename(args.model_path).split(".")[0]))
with open(rouge_output_file, 'w', encoding="utf-8") as f:
f.write(rouge_score_formatted)
f.write("Similarity score(sbert): %.3f\n" % similarity_score)
f.write(rouge_table + " & %.3f" % (similarity_score))
def cnndm_test(args, model, logger):
#Init model
model = AutoExtSummarizer(args)
model.to(args.device)
model.eval()
if "score" in args.label_key:
criterion = torch.nn.BCELoss(reduction='sum')
else:
criterion = torch.nn.MSELoss(reduction='sum')
tokenizer = AutoTokenizer.from_pretrained(args.model_name)
eval_dataset = CNNDMBlob(prefix='test',
data_path=args.data_dir,
tokenizer=tokenizer)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset,
sampler=eval_sampler,
batch_size=args.batch_size,
collate_fn=functools.partial(
collate,
pad_token_id=tokenizer.pad_token_id,
is_test=True),
num_workers=args.num_workers,
pin_memory=True)
logger.info("***** Running CNNDM evaluation *****")
logger.info(" Num examples = %d", len(eval_dataset))
eval_loss = 0.0
nb_eval_steps = 0
gold = []
pred = []
avg_loss = 0
for batch in tqdm(eval_dataloader, desc="Evaluating"): #, disable=True):
source = batch.src.to(args.device)
encoder_mask = batch.mask.to(args.device)
scores = batch.labels.to(args.device)
clss = batch.clss.to(args.device)
cls_mask = batch.mask_cls.to(args.device).bool()
target = batch.labels.to(args.device)
with torch.no_grad():
sent_scores, mask, memory = model(
source,
encoder_mask,
clss,
cls_mask,
)
loss = criterion(sent_scores, target.float())
avg_loss += loss.item()
#Seperates padding from the ones that are actually 0
sent_scores = sent_scores + mask.float()
sent_scores = sent_scores.cpu().data.numpy()
#Returns indexes sorted by descending score
selected_ids = np.argsort(-sent_scores, 1)
for idx, row in enumerate(selected_ids):
_pred = []
for i in row:
if i >= len(batch.src_str[idx]):
continue
candidate = batch.src_str[idx][i].strip()
if args.block_tri:
if (not _block_tri(candidate, _pred)):
_pred.append(candidate)
else:
_pred.append(candidate)
if len(_pred) == 3:
break
pred.append(_pred)
_gold = batch.tgt_str[idx]
gold.append(_gold)
#break
simple_rouge_score = []
for i in range(len(gold)):
simple_rouge_score.append(simple_rouge(pred[i], gold[i]))
simple_rouge_score = np.mean(simple_rouge_score) * 100
avg_loss = avg_loss / (len(eval_dataloader) * args.batch_size)
#python rouge implementation
rouge = Rouge()
rouge_score = rouge.get_scores([" ".join(p) for p in pred],
[" ".join(g) for g in gold],
avg=True)
rouge_score_formatted = format_rouge_scores(rouge_score)
rouge_table = format_rouge_table(rouge_score)
similarity_score = calc_sbert_similarity(pred, gold)
print(rouge_score_formatted)
print("Simple rouge score: %.3f" % simple_rouge_score)
print("Avg loss: %.3f" % avg_loss)
print("Similarity score(sbert): %.3f" % similarity_score)
print(rouge_table + " & %.3f & %.3f" %
(simple_rouge_score, similarity_score))
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
rouge_output_file = os.path.join(
args.output_dir, "cnndm_test_results_{}_{}.txt".format(
args.model_name,
os.path.basename(args.model_path).split(".")[0]))
with open(rouge_output_file, 'w', encoding="utf-8") as f:
f.write(rouge_score_formatted)
f.write("\nSimple rouge score: %.3f\n" % simple_rouge_score)
f.write("Avg loss: %.3f\n" % avg_loss)
f.write("Similarity score(sbert): %.3f\n" % similarity_score)
f.write(rouge_table + " & %.3f & %.3f" %
(simple_rouge_score, similarity_score))
def cnndm_label_ref(args, logger):
tokenizer = AutoTokenizer.from_pretrained(args.model_name)
eval_dataset = CNNDMBlobNoTokens(prefix='test',
data_path=args.data_dir,
tokenizer=tokenizer,
label_key=args.label_key,
max_pos=9999)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset,
sampler=eval_sampler,
batch_size=args.batch_size,
collate_fn=functools.partial(
collate,
pad_token_id=tokenizer.pad_token_id,
is_test=True),
num_workers=args.num_workers)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_dataset))
gold = []
lead3pred = []
greedyselectpred = []
for batch in tqdm(eval_dataloader, desc="Evaluating"):
source = batch.src
encoder_mask = batch.mask
scores = batch.labels
clss = batch.clss
cls_mask = batch.mask_cls
scores = np.argsort(-scores, 1)
for idx, row in enumerate(scores):
_pred = []
for jdx, i in enumerate(row):
if batch.labels[idx][i] == 0:
break
if i >= len(batch.src_str[idx]):
continue
candidate = batch.src_str[idx][i].strip()
_pred.append(candidate)
if len(_pred) == 3:
break
greedyselectpred.append(_pred)
lead3pred.append(batch.src_str[idx][:3])
gold.append(batch.tgt_str[idx])
rouge = Rouge()
simple_rouge_score_greed = np.mean(
[simple_rouge(greedyselectpred[i], gold[i])
for i in range(len(gold))]) * 100
rouge_score_greed = format_rouge_scores(
rouge.get_scores([" ".join(p) for p in greedyselectpred],
[" ".join(g) for g in gold],
avg=True))
similarity_score_greed = calc_sbert_similarity(greedyselectpred, gold)
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
rouge_output_file = os.path.join(args.output_dir, "cnndm_ref_results.txt")
with open(rouge_output_file, 'w', encoding="utf-8") as f:
f.write("GREED")
f.write(rouge_score_greed)
f.write("\nSimple rouge score: %f\n" % simple_rouge_score_greed)
f.write("Similarity score(sbert): %f\n" % similarity_score_greed)