def manual_eval(): # Load vocabularies. sc_vocab_path = os.path.join(FLAGS.data_dir, "vocab%d.nl" % FLAGS.sc_vocab_size) tg_vocab_path = os.path.join(FLAGS.data_dir, "vocab%d.cm.ast" % FLAGS.tg_vocab_size) sc_vocab, rev_sc_vocab = data_utils.initialize_vocabulary(sc_vocab_path) tg_vocab, rev_tg_vocab = data_utils.initialize_vocabulary(tg_vocab_path) train_set, dev_set, test_set = load_data() model = knn_model.KNNModel() model.train(train_set) eval_tools.manual_eval(model_name, test_set, rev_sc_vocab, FLAGS, FLAGS.model_dir, num_eval=500)
def manual_eval(dataset, model_dir=None, decode_sig=None): if model_dir is None: model_subdir, decode_sig = graph_utils.get_decode_signature(FLAGS) model_dir = os.path.join(FLAGS.model_root_dir, model_subdir) print("(Manual) evaluating " + model_dir) return eval_tools.manual_eval(model_dir, decode_sig, dataset, FLAGS, top_k=3, num_examples=100, verbose=True)
def manual_eval(dataset, prediction_path=None): if prediction_path is None: model_subdir, decode_sig = graph_utils.get_decode_signature(FLAGS) model_dir = os.path.join(FLAGS.model_root_dir, model_subdir) prediction_path = os.path.join(model_dir, 'predictions.{}.latest'.format(decode_sig)) print("(Manual) evaluating " + prediction_path) return eval_tools.manual_eval(prediction_path, dataset, FLAGS, top_k=3, num_examples=100, verbose=True)
def manual_eval(dataset, num_eval): _, decode_sig = graph_utils.get_decode_signature(FLAGS) eval_tools.manual_eval(decode_sig, dataset, FLAGS, FLAGS.model_root_dir, num_eval)