def validate(model, test_data, golden_file, beam_size=8, alpha=0.6, max_time_step=100): """For development Only""" pp = PostProcess() ref_stream = [] for line in open(golden_file + '.input_clean'): if line.startswith('# ::tokens '): o = json.loads(line[len('# ::tokens '):].strip()) ref_stream.append(' '.join(o).lower()) # gold model output graph, gold_sys_stream, _, abstract = read_file(golden_file + '.preproc') ref_streams = [ref_stream] sys_stream = [] for batch in test_data: res = generate_batch(model, batch, beam_size, alpha, max_time_step) sys_stream.extend(res['token']) assert len(sys_stream) == len(ref_stream) sys_stream = [ pp.post_process(o, abstract[i], graph[i]) for i, o in enumerate(sys_stream) ] bleu = sacrebleu.corpus_bleu(sys_stream, ref_streams, force=True, lowercase=True, tokenize='none').score chrf = sacrebleu.corpus_chrf(sys_stream, ref_stream) return bleu, chrf
def validate(model, test_data, beam_size=8, alpha=0.6, max_time_step=100): """For development Only""" pp = PostProcess() ref_stream = [] sys_stream = [] for batch in test_data: res = generate_batch(model, batch, beam_size, alpha, max_time_step) sys_stream.extend(res['token']) ref_stream.extend(batch['target']) assert len(sys_stream) == len(ref_stream) sys_stream = [pp.post_process(o) for o in sys_stream] ref_stream = [' '.join(o) for i in ref_stream] ref_streams = [ref_stream] bleu = sacrebleu.corpus_bleu(sys_stream, ref_streams, force=True, lowercase=False, tokenize='none').score chrf = sacrebleu.corpus_chrf(sys_stream, ref_stream) return bleu, chrf