if args.operation == 'evaluate':
dataset = eval(args.type)
if config.mode == 'self':
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
evaluate_decode_results(dataset, decode_results)
elif config.mode == 'seq2tree':
from evaluation import evaluate_seq2tree_sample_file
evaluate_seq2tree_sample_file(config.seq2tree_sample_file,
config.seq2tree_id_file, dataset)
elif config.mode == 'seq2seq':
from evaluation import evaluate_seq2seq_decode_results
evaluate_seq2seq_decode_results(dataset,
config.seq2seq_decode_file,
config.seq2seq_ref_file,
is_nbest=config.is_nbest)
elif config.mode == 'analyze':
from evaluation import analyze_decode_results
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
analyze_decode_results(dataset, decode_results)
if args.operation == 'interactive':
from dataset import canonicalize_query, query_to_data
from collections import namedtuple
from lang.py.parse import decode_tree_to_python_ast
assert model is not None
while True:
serialize_to_file(decode_results, args.saveto)
if args.operation == 'evaluate':
dataset = eval(args.type)
if config.mode == 'self':
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
evaluate_decode_results(dataset, decode_results)
elif config.mode == 'seq2tree':
from evaluation import evaluate_seq2tree_sample_file
evaluate_seq2tree_sample_file(config.seq2tree_sample_file, config.seq2tree_id_file, dataset)
elif config.mode == 'seq2seq':
from evaluation import evaluate_seq2seq_decode_results
evaluate_seq2seq_decode_results(dataset, config.seq2seq_decode_file, config.seq2seq_ref_file, is_nbest=config.is_nbest)
elif config.mode == 'analyze':
from evaluation import analyze_decode_results
decode_results_file = args.input
decode_results = deserialize_from_file(decode_results_file)
analyze_decode_results(dataset, decode_results)
if args.operation == 'interactive':
from dataset import canonicalize_query, query_to_data
from collections import namedtuple
from lang.py.parse import decode_tree_to_python_ast
assert model is not None
while True:
cmd = raw_input('example id or query: ')