def eval_model(model,outfilename, word_to_ix, all_tags=None,trainfile=TRAIN_FILE,testfile=DEV_FILE): """Calculate confusion_matrix for a given model Parameters: tagger -- Model mapping (words) to an optimal sequence of tags for the words outfilename -- Filename to write tagger predictions to testfile -- (optional) Filename containing true labels Returns: confusion_matrix -- dict of occurences of (true_label, pred_label) """ apply_model(model,outfilename,word_to_ix, all_tags,trainfile,testfile) return scorer.get_confusion(testfile,outfilename) #run the scorer on the prediction file
def test_nr_hmm_test_accuracy(): confusion = scorer.get_confusion(NR_TEST_FILE, 'hmm-te-nr.preds') acc = scorer.accuracy(confusion) ok_(acc > .853)
def test_nr_hmm_dev_accuracy(): confusion = scorer.get_confusion(NR_DEV_FILE, 'hmm-dev-nr.preds') acc = scorer.accuracy(confusion) ok_(acc > .861)
def test_hmm_test_accuracy(): confusion = scorer.get_confusion(TEST_FILE, 'hmm-te-en.preds') acc = scorer.accuracy(confusion) ok_(acc > .840)
def test_hmm_dev_accuracy(): confusion = scorer.get_confusion(DEV_FILE, 'hmm-dev-en.preds') acc = scorer.accuracy(confusion) ok_(acc > .840)
def test_nr_test_accuracy3(): confusion = scorer.get_confusion(NR_TEST_FILE, 'bakeoff-te-nr.preds') acc = scorer.accuracy(confusion) ok_(acc > .86)
def test_en_dev_accuracy1(): confusion = scorer.get_confusion(DEV_FILE, 'bakeoff-dev-en.preds') acc = scorer.accuracy(confusion) ok_(acc > .85)
def test_nr_dev_accuracy3(): confusion = scorer.get_confusion(NR_DEV_FILE, 'bakeoff-dev-nr.preds') acc = scorer.accuracy(confusion) ok_(acc > .87)
def test_en_test_accuracy3(): confusion = scorer.get_confusion(TEST_FILE, 'bakeoff-te-en.preds') acc = scorer.accuracy(confusion) ok_(acc > .86)
def test_bilstm_dev_accuracy(): confusion = scorer.get_confusion(DEV_FILE, 'bilstm-dev-en.preds') acc = scorer.accuracy(confusion) ok_(acc > .85)