def evaluate_generation(generator,
data_iter,
save_file=None,
num_batches=None,
verbos=False,
embMetric=False,):
"""
evaluate_generation
"""
results = generator.generate(batch_iter=data_iter,
num_batches=num_batches)
refs = [result.tgt.split(" ") for result in results]
hyps = [result.preds[0].split(" ") for result in results]
report_message = []
avg_len = np.average([len(s) for s in hyps])
report_message.append("Avg_Len-{:.3f}".format(avg_len))
bleu_1, bleu_2 = bleu(hyps, refs)
report_message.append("Bleu-{:.4f}/{:.4f}".format(bleu_1, bleu_2))
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps)
report_message.append("Inter_Dist-{:.4f}/{:.4f}".format(inter_dist1, inter_dist2))
report_message.append("Intra_Dist-{:.4f}/{:.4f}".format(intra_dist1, intra_dist2))
# embedding metrics
# embed_metric = EmbeddingMetrics(field=generator.tgt_field)
# ext_sim, avg_sim, greedy_sim = embed_metric.embed_sim(
# hyp_texts=[' '.join(ws) for ws in hyps],
# ref_texts=[' '.join(ws) for ws in refs])
# report_message.append(
# f"Embed(E/A/G)-{ext_sim:.4f}/{avg_sim:.4f}/{greedy_sim:.4f}")
report_message = " ".join(report_message)
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs)
avg_len = np.average([len(s) for s in refs])
target_message = "Target: AVG_LEN-{:.3f} ".format(avg_len) + \
"Inter_Dist-{:.4f}/{:.4f}".format(inter_dist1, inter_dist2) + " Intra_Dist-{:.4f}/{:.4f}".format(
intra_dist1, intra_dist2)
message = report_message + "\n" + target_message
if save_file is not None:
write_results(results, save_file)
print("Saved generation results to '{}'".format(save_file))
if verbos:
print(message)
else:
return message
def evaluate_generation(generator,
data_iter,
num_candidates=5,
save_file=None,
num_batches=None,
verbos=False):
results = generator.generate(batch_iter=data_iter,
num_candidates=num_candidates,
num_batches=num_batches)
refs = [result.tgt.split(" ") for result in results]
hyps = [result.preds[0].split(" ") for result in results]
report_message = []
avg_len = np.average([len(s) for s in hyps])
report_message.append(f"Avg_Len-{avg_len:.3f}")
bleu_1, bleu_2 = bleu(hyps, refs)
report_message.append(f"Bleu-{bleu_1:.4f}/{bleu_2:.4f}")
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps)
report_message.append(f"Inter_Dist-{inter_dist1:.4f}/{inter_dist2:.4f}")
# embed_metric = EmbeddingMetrics(field=generator.tgt_field)
# ext_sim, avg_sim, greedy_sim = embed_metric.embed_sim(
# hyp_texts=[' '.join(ws) for ws in hyps],
# ref_texts=[' '.join(ws) for ws in refs])
# report_message.append(
# f"Embed(E/A/G)-{ext_sim:.4f}/{avg_sim:.4f}/{greedy_sim:.4f}")
report_message = " ".join(report_message)
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs)
avg_len = np.average([len(s) for s in refs])
target_message = f"Target: AVG_LEN-{avg_len:.3f} " + \
f"Inter_Dist-{inter_dist1:.4f}/{inter_dist2:.4f}"
message = report_message + "\n" + target_message
if save_file is not None:
write_results(results, save_file)
print(f"Saved generation results to '{save_file}'")
if verbos:
print(message)
else:
return message
def generate(self, data_iter, save_file=None, verbos=False):
"""
generate
"""
results = []
num_batches = data_iter.n_batch
for batch_idx in range(num_batches):
local_data = data_iter.get_batch(batch_idx)
turn_inputs = create_turn_batch(local_data['inputs'])
kb_inputs = create_kb_batch(local_data['kbs'])
assert len(turn_inputs) == local_data['max_turn']
result_batch = self.forward(turn_inputs=turn_inputs,
kb_inputs=kb_inputs,
enc_hidden=None)
results.append(result_batch)
# post processing the results
hyps = []
refs = []
turn_labels = []
tasks = []
gold_entity = []
kb_word = []
for result_batch in results:
for result_turn in result_batch:
hyps.append(result_turn.preds[0].split(" "))
refs.append(result_turn.tgt.split(" "))
turn_labels.append(result_turn.turn_label)
tasks.append(result_turn.task)
gold_entity.append(result_turn.gold_entity)
kb_word.append(result_turn.kb_word)
outputs = []
for tl, hyp, ref, tk, ent, kb in zip(turn_labels, hyps, refs, tasks,
gold_entity, kb_word):
hyp_str = " ".join(hyp)
ref_str = " ".join(ref)
sample = {
"turn_label": str(tl),
"result": hyp_str,
"target": ref_str,
"task": tk,
"gold_entity": ent,
"kb": kb
}
outputs.append(sample)
avg_len = np.average([len(s) for s in hyps])
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps)
report_message = "Result: Avg_Len={:.3f} ".format(avg_len) + \
"Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2)
avg_len = np.average([len(s) for s in refs])
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs)
target_message = "Target: Avg_Len={:.3f} ".format(avg_len) + \
"Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2)
message = target_message + "\n" + report_message
if verbos:
print(message)
if save_file is not None:
with open(save_file, 'w', encoding='utf-8') as fw:
for sample in outputs:
line = json.dumps(sample)
fw.write(line)
fw.write('\n')
print("Saved generation results to '{}.'".format(save_file))
def generate(self, data_iter, save_file=None, verbos=False):
"""
generate
"""
results = []
num_batches = data_iter.n_batch
for batch_idx in tqdm(range(num_batches)):
local_data = data_iter.get_batch(batch_idx)
turn_inputs = create_turn_batch(local_data['inputs'])
kb_inputs = create_kb_batch(local_data['kbs'])
situation_inputs = create_situation_batch(local_data['situation'])
user_profile_inputs = create_user_profile_batch(
local_data['user_profile'])
assert len(turn_inputs) == local_data['max_turn']
result_batch = self.forward(
turn_inputs=turn_inputs,
kb_inputs=kb_inputs,
situation_inputs=situation_inputs,
user_profile_inputs=user_profile_inputs,
enc_hidden=None)
results.append(result_batch)
# post processing the results
srcs = []
hyps = []
refs = []
# turn_labels = []
tasks = []
gold_entity = []
# kb_word = []
for result_batch in results:
for result_turn in result_batch:
srcs.append(result_turn.src.split(" "))
hyps.append(result_turn.preds[0].split(" "))
refs.append(result_turn.tgt.split(" "))
# turn_labels.append(result_turn.turn_label)
tasks.append(result_turn.task)
gold_entity.append(result_turn.gold_entity)
# kb_word.append(result_turn.kb_word)
outputs = []
for src, hyp, ref, tk, ent in zip(srcs, hyps, refs, tasks,
gold_entity):
src_str = " ".join(src)
hyp_str = " ".join(hyp)
ref_str = " ".join(ref)
if self.mode == 'test':
sample = {
"source": src_str,
"result": hyp_str,
"target": ref_str
}
else:
sample = {
"result": hyp_str,
"target": ref_str,
"task": tk,
"gold_entity": ent
}
outputs.append(sample)
avg_len = np.average([len(s) for s in hyps])
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps)
report_message = "Result: Avg_Len={:.3f} ".format(avg_len) + \
"Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2)
if self.mode != 'test':
avg_len = np.average([len(s) for s in refs])
intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs)
target_message = "Target: Avg_Len={:.3f} ".format(avg_len) + \
"Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2)
else:
target_message = "Target: Avg_Len={:.3f} ".format(1) + \
"Inter_Dist={:.4f}/{:.4f}".format(0.0, 0.0)
message = target_message + "\n" + report_message
if verbos:
print(message)
if save_file is not None:
with open(save_file, 'w', encoding='utf-8') as fw:
for sample in outputs:
line = json.dumps(sample)
fw.write(line)
fw.write('\n')
print("Saved generation results to '{}.'".format(save_file))
