def write_trees_or_tokens(output_file, das, gen_trees, base_doc, language, selector):
"""Decide to write t-trees or tokens based on the output file name."""
if output_file.endswith('.txt'):
gen_toks = [t.to_tok_list() for t in gen_trees]
postprocess_tokens(gen_toks, das)
write_tokens(gen_toks, output_file)
else:
write_ttrees(create_ttree_doc(gen_trees, base_doc, language, selector), output_file)
def eval_tokens(das, eval_tokens, gen_tokens):
"""Evaluate generated tokens and print out statistics."""
postprocess_tokens(eval_tokens, das)
postprocess_tokens(gen_tokens, das)
evaluator = BLEUMeasure()
for pred_sent, gold_sents in zip(gen_tokens, eval_tokens):
evaluator.append(pred_sent, gold_sents)
log_info("BLEU score: %.4f" % (evaluator.bleu() * 100))
evaluator = Evaluator()
for pred_sent, gold_sents in zip(gen_tokens, eval_tokens):
for gold_sent in gold_sents: # effectively an average over all gold paraphrases
evaluator.append(gold_sent, pred_sent)
log_info("TOKEN precision: %.4f, Recall: %.4f, F1: %.4f" % evaluator.p_r_f1(EvalTypes.TOKEN))
log_info("Sentence length stats:\n * GOLD %s\n * PRED %s\n * DIFF %s" % evaluator.size_stats())
log_info("Common subphrase stats:\n -- SIZE: %s\n -- ΔGLD: %s\n -- ΔPRD: %s" %
evaluator.common_substruct_stats())
def eval_tokens(das, eval_tokens, gen_tokens):
"""Evaluate generated tokens and print out statistics."""
postprocess_tokens(eval_tokens, das)
postprocess_tokens(gen_tokens, das)
evaluator = BLEUMeasure()
for pred_sent, gold_sents in zip(gen_tokens, eval_tokens):
evaluator.append(pred_sent, gold_sents)
log_info("BLEU score: %.4f" % (evaluator.bleu() * 100))
evaluator = Evaluator()
for pred_sent, gold_sents in zip(gen_tokens, eval_tokens):
for gold_sent in gold_sents: # effectively an average over all gold paraphrases
evaluator.append(gold_sent, pred_sent)
log_info("TOKEN precision: %.4f, Recall: %.4f, F1: %.4f" % evaluator.p_r_f1(EvalTypes.TOKEN))
log_info("Sentence length stats:\n * GOLD %s\n * PRED %s\n * DIFF %s" % evaluator.size_stats())
log_info("Common subphrase stats:\n -- SIZE: %s\n -- ΔGLD: %s\n -- ΔPRD: %s" %
evaluator.common_substruct_stats())
def seq2seq_gen(args):
"""Sequence-to-sequence generation"""
ap = ArgumentParser()
ap.add_argument('-e',
'--eval-file',
type=str,
help='A ttree/text file for evaluation')
ap.add_argument(
'-a',
'--abstr-file',
type=str,
help=
'Lexicalization file (a.k.a. abstraction instructions, for postprocessing)'
)
ap.add_argument('-r',
'--ref-selector',
type=str,
default='',
help='Selector for reference trees in the evaluation file')
ap.add_argument(
'-t',
'--target-selector',
type=str,
default='',
help='Target selector for generated trees in the output file')
ap.add_argument('-d',
'--debug-logfile',
type=str,
help='Debug output file name')
ap.add_argument('-w',
'--output-file',
type=str,
help='Output tree/text file')
ap.add_argument('-b',
'--beam-size',
type=int,
help='Override beam size for beam search decoding')
ap.add_argument('-c',
'--context-file',
type=str,
help='Input ttree/text file with context utterances')
ap.add_argument('seq2seq_model_file',
type=str,
help='Trained Seq2Seq generator model')
ap.add_argument('da_test_file', type=str, help='Input DAs for generation')
args = ap.parse_args(args)
if args.debug_logfile:
set_debug_stream(file_stream(args.debug_logfile, mode='w'))
# load the generator
tgen = Seq2SeqBase.load_from_file(args.seq2seq_model_file)
if args.beam_size is not None:
tgen.beam_size = args.beam_size
# read input files
das = read_das(args.da_test_file)
if args.context_file:
if not tgen.use_context and not tgen.context_bleu_weight:
log_warn(
'Generator is not trained to use context, ignoring context input file.'
)
else:
if args.context_file.endswith('.txt'):
contexts = read_tokens(args.context_file)
else:
contexts = tokens_from_doc(read_ttrees(args.context_file),
tgen.language, tgen.selector)
das = [(context, da) for context, da in zip(contexts, das)]
# generate
log_info('Generating...')
gen_trees = []
for num, da in enumerate(das, start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_trees.append(tgen.generate_tree(da))
log_info(tgen.get_slot_err_stats())
# evaluate the generated trees against golden trees (delexicalized)
eval_doc = None
if args.eval_file and not args.eval_file.endswith('.txt'):
eval_doc = read_ttrees(args.eval_file)
evaler = Evaluator()
evaler.process_eval_doc(eval_doc, gen_trees, tgen.language,
args.ref_selector, args.target_selector
or tgen.selector)
# lexicalize, if required
if args.abstr_file and tgen.lexicalizer:
log_info('Lexicalizing...')
tgen.lexicalize(gen_trees, args.abstr_file)
# evaluate the generated & lexicalized tokens (F1 and BLEU scores)
if args.eval_file and args.eval_file.endswith('.txt'):
eval_tokens(das, read_tokens(args.eval_file, ref_mode=True),
[t.to_tok_list() for t in gen_trees])
# write output .yaml.gz or .txt
if args.output_file is not None:
log_info('Writing output...')
if args.output_file.endswith('.txt'):
write_tokens(
postprocess_tokens([t.to_tok_list() for t in gen_trees], das),
args.output_file)
else:
write_ttrees(
create_ttree_doc(gen_trees, eval_doc, tgen.language,
args.target_selector or tgen.selector),
args.output_file)
def seq2seq_gen(args):
"""Sequence-to-sequence generation"""
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument('-e', '--eval-file', type=str, help='A ttree/text file for evaluation')
ap.add_argument('-a', '--abstr-file', type=str,
help='Lexicalization file (a.k.a. abstraction instructions, for postprocessing)')
ap.add_argument('-r', '--ref-selector', type=str, default='',
help='Selector for reference trees in the evaluation file')
ap.add_argument('-t', '--target-selector', type=str, default='',
help='Target selector for generated trees in the output file')
ap.add_argument('-d', '--debug-logfile', type=str, help='Debug output file name')
ap.add_argument('-w', '--output-file', type=str, help='Output tree/text file')
ap.add_argument('-b', '--beam-size', type=int,
help='Override beam size for beam search decoding')
ap.add_argument('-c', '--context-file', type=str,
help='Input ttree/text file with context utterances')
ap.add_argument('seq2seq_model_file', type=str, help='Trained Seq2Seq generator model')
ap.add_argument('da_test_file', type=str, help='Input DAs for generation')
args = ap.parse_args(args)
if args.debug_logfile:
set_debug_stream(file_stream(args.debug_logfile, mode='w'))
# load the generator
tgen = Seq2SeqBase.load_from_file(args.seq2seq_model_file)
if args.beam_size is not None:
tgen.beam_size = args.beam_size
# read input files (DAs, contexts)
das = read_das(args.da_test_file)
if args.context_file:
if not tgen.use_context and not tgen.context_bleu_weight:
log_warn('Generator is not trained to use context, ignoring context input file.')
else:
if args.context_file.endswith('.txt'):
contexts = read_tokens(args.context_file)
else:
contexts = tokens_from_doc(read_ttrees(args.context_file),
tgen.language, tgen.selector)
das = [(context, da) for context, da in zip(contexts, das)]
elif tgen.use_context or tgen.context_bleu_weight:
log_warn('Generator is trained to use context. ' +
'Using empty contexts, expect lower performance.')
das = [([], da) for da in das]
# generate
log_info('Generating...')
gen_trees = []
for num, da in enumerate(das, start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_trees.append(tgen.generate_tree(da))
if num % 100 == 0:
log_info("Generated tree %d" % num)
log_info(tgen.get_slot_err_stats())
# evaluate the generated trees against golden trees (delexicalized)
eval_doc = None
if args.eval_file and not args.eval_file.endswith('.txt'):
eval_doc = read_ttrees(args.eval_file)
evaler = Evaluator()
evaler.process_eval_doc(eval_doc, gen_trees, tgen.language, args.ref_selector,
args.target_selector or tgen.selector)
# lexicalize, if required
if args.abstr_file and tgen.lexicalizer:
log_info('Lexicalizing...')
tgen.lexicalize(gen_trees, args.abstr_file)
# we won't need contexts anymore, but we do need DAs
if tgen.use_context or tgen.context_bleu_weight:
das = [da for _, da in das]
# evaluate the generated & lexicalized tokens (F1 and BLEU scores)
if args.eval_file and args.eval_file.endswith('.txt'):
eval_tokens(das, read_tokens(args.eval_file, ref_mode=True),
[t.to_tok_list() for t in gen_trees])
# write output .yaml.gz or .txt
if args.output_file is not None:
log_info('Writing output...')
if args.output_file.endswith('.txt'):
gen_toks = [t.to_tok_list() for t in gen_trees]
postprocess_tokens(gen_toks, das)
write_tokens(gen_toks, args.output_file)
else:
write_ttrees(create_ttree_doc(gen_trees, eval_doc, tgen.language,
args.target_selector or tgen.selector),
args.output_file)