def evaluate_generation(generator, data_iter, save_file=None, num_batches=None, verbos=False, embMetric=False,): """ evaluate_generation """ results = generator.generate(batch_iter=data_iter, num_batches=num_batches) refs = [result.tgt.split(" ") for result in results] hyps = [result.preds[0].split(" ") for result in results] report_message = [] avg_len = np.average([len(s) for s in hyps]) report_message.append("Avg_Len-{:.3f}".format(avg_len)) bleu_1, bleu_2 = bleu(hyps, refs) report_message.append("Bleu-{:.4f}/{:.4f}".format(bleu_1, bleu_2)) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps) report_message.append("Inter_Dist-{:.4f}/{:.4f}".format(inter_dist1, inter_dist2)) report_message.append("Intra_Dist-{:.4f}/{:.4f}".format(intra_dist1, intra_dist2)) # embedding metrics # embed_metric = EmbeddingMetrics(field=generator.tgt_field) # ext_sim, avg_sim, greedy_sim = embed_metric.embed_sim( # hyp_texts=[' '.join(ws) for ws in hyps], # ref_texts=[' '.join(ws) for ws in refs]) # report_message.append( # f"Embed(E/A/G)-{ext_sim:.4f}/{avg_sim:.4f}/{greedy_sim:.4f}") report_message = " ".join(report_message) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs) avg_len = np.average([len(s) for s in refs]) target_message = "Target: AVG_LEN-{:.3f} ".format(avg_len) + \ "Inter_Dist-{:.4f}/{:.4f}".format(inter_dist1, inter_dist2) + " Intra_Dist-{:.4f}/{:.4f}".format( intra_dist1, intra_dist2) message = report_message + "\n" + target_message if save_file is not None: write_results(results, save_file) print("Saved generation results to '{}'".format(save_file)) if verbos: print(message) else: return message
def evaluate_generation(generator, data_iter, num_candidates=5, save_file=None, num_batches=None, verbos=False): results = generator.generate(batch_iter=data_iter, num_candidates=num_candidates, num_batches=num_batches) refs = [result.tgt.split(" ") for result in results] hyps = [result.preds[0].split(" ") for result in results] report_message = [] avg_len = np.average([len(s) for s in hyps]) report_message.append(f"Avg_Len-{avg_len:.3f}") bleu_1, bleu_2 = bleu(hyps, refs) report_message.append(f"Bleu-{bleu_1:.4f}/{bleu_2:.4f}") intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps) report_message.append(f"Inter_Dist-{inter_dist1:.4f}/{inter_dist2:.4f}") # embed_metric = EmbeddingMetrics(field=generator.tgt_field) # ext_sim, avg_sim, greedy_sim = embed_metric.embed_sim( # hyp_texts=[' '.join(ws) for ws in hyps], # ref_texts=[' '.join(ws) for ws in refs]) # report_message.append( # f"Embed(E/A/G)-{ext_sim:.4f}/{avg_sim:.4f}/{greedy_sim:.4f}") report_message = " ".join(report_message) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs) avg_len = np.average([len(s) for s in refs]) target_message = f"Target: AVG_LEN-{avg_len:.3f} " + \ f"Inter_Dist-{inter_dist1:.4f}/{inter_dist2:.4f}" message = report_message + "\n" + target_message if save_file is not None: write_results(results, save_file) print(f"Saved generation results to '{save_file}'") if verbos: print(message) else: return message
def generate(self, data_iter, save_file=None, verbos=False): """ generate """ results = [] num_batches = data_iter.n_batch for batch_idx in range(num_batches): local_data = data_iter.get_batch(batch_idx) turn_inputs = create_turn_batch(local_data['inputs']) kb_inputs = create_kb_batch(local_data['kbs']) assert len(turn_inputs) == local_data['max_turn'] result_batch = self.forward(turn_inputs=turn_inputs, kb_inputs=kb_inputs, enc_hidden=None) results.append(result_batch) # post processing the results hyps = [] refs = [] turn_labels = [] tasks = [] gold_entity = [] kb_word = [] for result_batch in results: for result_turn in result_batch: hyps.append(result_turn.preds[0].split(" ")) refs.append(result_turn.tgt.split(" ")) turn_labels.append(result_turn.turn_label) tasks.append(result_turn.task) gold_entity.append(result_turn.gold_entity) kb_word.append(result_turn.kb_word) outputs = [] for tl, hyp, ref, tk, ent, kb in zip(turn_labels, hyps, refs, tasks, gold_entity, kb_word): hyp_str = " ".join(hyp) ref_str = " ".join(ref) sample = { "turn_label": str(tl), "result": hyp_str, "target": ref_str, "task": tk, "gold_entity": ent, "kb": kb } outputs.append(sample) avg_len = np.average([len(s) for s in hyps]) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps) report_message = "Result: Avg_Len={:.3f} ".format(avg_len) + \ "Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2) avg_len = np.average([len(s) for s in refs]) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs) target_message = "Target: Avg_Len={:.3f} ".format(avg_len) + \ "Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2) message = target_message + "\n" + report_message if verbos: print(message) if save_file is not None: with open(save_file, 'w', encoding='utf-8') as fw: for sample in outputs: line = json.dumps(sample) fw.write(line) fw.write('\n') print("Saved generation results to '{}.'".format(save_file))
def generate(self, data_iter, save_file=None, verbos=False): """ generate """ results = [] num_batches = data_iter.n_batch for batch_idx in tqdm(range(num_batches)): local_data = data_iter.get_batch(batch_idx) turn_inputs = create_turn_batch(local_data['inputs']) kb_inputs = create_kb_batch(local_data['kbs']) situation_inputs = create_situation_batch(local_data['situation']) user_profile_inputs = create_user_profile_batch( local_data['user_profile']) assert len(turn_inputs) == local_data['max_turn'] result_batch = self.forward( turn_inputs=turn_inputs, kb_inputs=kb_inputs, situation_inputs=situation_inputs, user_profile_inputs=user_profile_inputs, enc_hidden=None) results.append(result_batch) # post processing the results srcs = [] hyps = [] refs = [] # turn_labels = [] tasks = [] gold_entity = [] # kb_word = [] for result_batch in results: for result_turn in result_batch: srcs.append(result_turn.src.split(" ")) hyps.append(result_turn.preds[0].split(" ")) refs.append(result_turn.tgt.split(" ")) # turn_labels.append(result_turn.turn_label) tasks.append(result_turn.task) gold_entity.append(result_turn.gold_entity) # kb_word.append(result_turn.kb_word) outputs = [] for src, hyp, ref, tk, ent in zip(srcs, hyps, refs, tasks, gold_entity): src_str = " ".join(src) hyp_str = " ".join(hyp) ref_str = " ".join(ref) if self.mode == 'test': sample = { "source": src_str, "result": hyp_str, "target": ref_str } else: sample = { "result": hyp_str, "target": ref_str, "task": tk, "gold_entity": ent } outputs.append(sample) avg_len = np.average([len(s) for s in hyps]) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(hyps) report_message = "Result: Avg_Len={:.3f} ".format(avg_len) + \ "Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2) if self.mode != 'test': avg_len = np.average([len(s) for s in refs]) intra_dist1, intra_dist2, inter_dist1, inter_dist2 = distinct(refs) target_message = "Target: Avg_Len={:.3f} ".format(avg_len) + \ "Inter_Dist={:.4f}/{:.4f}".format(inter_dist1, inter_dist2) else: target_message = "Target: Avg_Len={:.3f} ".format(1) + \ "Inter_Dist={:.4f}/{:.4f}".format(0.0, 0.0) message = target_message + "\n" + report_message if verbos: print(message) if save_file is not None: with open(save_file, 'w', encoding='utf-8') as fw: for sample in outputs: line = json.dumps(sample) fw.write(line) fw.write('\n') print("Saved generation results to '{}.'".format(save_file))