def evaluate_generation(generator,
data_iter,
save_file=None,
num_batches=None,
verbos=False,
embMetric=False,):
"""
evaluate_generation
"""
results = generator.generate(batch_iter=data_iter,
num_batches=num_batches)
refs = [result.tgt.split(" ") for result in results]
hyps = [result.preds[0].split(" ") for result in results]
report_message = []
avg_len = np.average([len(s) for s in hyps])
report_message.append("Avg_Len-{:.3f}".format(avg_len))
bleu_1, bleu_2 = bleu(hyps, refs)
report_message.append("Bleu-{:.4f}/{:.4f}".format(bleu_1, bleu_2))
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps)
report_message.append("Inter_Dist-{:.4f}/{:.4f}".format(inter_dist1, inter_dist2))
report_message.append("Intra_Dist-{:.4f}/{:.4f}".format(intra_dist1, intra_dist2))
# embedding metrics
# embed_metric = EmbeddingMetrics(field=generator.tgt_field)
# ext_sim, avg_sim, greedy_sim = embed_metric.embed_sim(
# hyp_texts=[' '.join(ws) for ws in hyps],
# ref_texts=[' '.join(ws) for ws in refs])
# report_message.append(
# f"Embed(E/A/G)-{ext_sim:.4f}/{avg_sim:.4f}/{greedy_sim:.4f}")
report_message = " ".join(report_message)
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs)
avg_len = np.average([len(s) for s in refs])
target_message = "Target: AVG_LEN-{:.3f} ".format(avg_len) + \
"Inter_Dist-{:.4f}/{:.4f}".format(inter_dist1, inter_dist2) + " Intra_Dist-{:.4f}/{:.4f}".format(
intra_dist1, intra_dist2)
message = report_message + "\n" + target_message
if save_file is not None:
write_results(results, save_file)
print("Saved generation results to '{}'".format(save_file))
if verbos:
print(message)
else:
return message
def evaluate_generation(generator,
data_iter,
num_candidates=5,
save_file=None,
num_batches=None,
verbos=False):
results = generator.generate(batch_iter=data_iter,
num_candidates=num_candidates,
num_batches=num_batches)
refs = [result.tgt.split(" ") for result in results]
hyps = [result.preds[0].split(" ") for result in results]
report_message = []
avg_len = np.average([len(s) for s in hyps])
report_message.append(f"Avg_Len-{avg_len:.3f}")
bleu_1, bleu_2 = bleu(hyps, refs)
report_message.append(f"Bleu-{bleu_1:.4f}/{bleu_2:.4f}")
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps)
report_message.append(f"Inter_Dist-{inter_dist1:.4f}/{inter_dist2:.4f}")
# embed_metric = EmbeddingMetrics(field=generator.tgt_field)
# ext_sim, avg_sim, greedy_sim = embed_metric.embed_sim(
# hyp_texts=[' '.join(ws) for ws in hyps],
# ref_texts=[' '.join(ws) for ws in refs])
# report_message.append(
# f"Embed(E/A/G)-{ext_sim:.4f}/{avg_sim:.4f}/{greedy_sim:.4f}")
report_message = " ".join(report_message)
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs)
avg_len = np.average([len(s) for s in refs])
target_message = f"Target: AVG_LEN-{avg_len:.3f} " + \
f"Inter_Dist-{inter_dist1:.4f}/{inter_dist2:.4f}"
message = report_message + "\n" + target_message
if save_file is not None:
write_results(results, save_file)
print(f"Saved generation results to '{save_file}'")
if verbos:
print(message)
else:
return message