def evaluate_on_test_set(test_essay_feats, out_predictions_file, out_predicted_margins_file, out_metrics_file, out_categories_file): test_feats, test_tags = flatten_to_wordlevel_feat_tags(test_essay_feats) wd_test_ys_bytag = get_wordlevel_ys_by_code(test_tags, wd_train_tags) test_x = feature_transformer.transform(test_feats) """ TEST Tagger """ test_wd_predictions_by_code = test_classifier_per_code(test_x, tag2word_classifier, wd_test_tags) print "\nRunning Sentence Model" """ SENTENCE LEVEL PREDICTIONS FROM STACKING """ sent_test_xs, sent_test_ys_bycode = get_sent_feature_for_stacking_from_tagging_model(sent_input_feat_tags, sent_input_interaction_tags, test_essay_feats, test_x, wd_test_ys_bytag, tag2word_classifier, SPARSE_SENT_FEATS, LOOK_BACK) """ Test Stack Classifier """ test_sent_predictions_by_code \ = test_classifier_per_code(sent_test_xs, tag2sent_classifier, sent_output_train_test_tags) if USE_SVM: test_decision_functions_by_code = test_classifier_per_code(sent_test_xs, tag2sent_classifier, sent_output_train_test_tags, predict_fn=decision_function_for_tag) else: test_decision_functions_by_code = test_classifier_per_code(sent_test_xs, tag2sent_classifier, sent_output_train_test_tags, predict_fn=probability_for_tag) """ Write out the predicted classes """ with open(out_predictions_file, "w+") as f_output_file: f_output_file.write("Essay|Sent Number|Processed Sentence|Concept Codes|Predictions\n") predictions_to_file(f_output_file, sent_test_ys_bycode, test_sent_predictions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags) with open(out_predicted_margins_file, "w+") as f_output_file: f_output_file.write("Essay|Sent Number|Processed Sentence|Concept Codes|Predicted Confidence\n") predictions_to_file(f_output_file, sent_test_ys_bycode, test_decision_functions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags, output_confidence=True) """ Write out the accuracy metrics """ train_wd_metrics = ResultsProcessor.compute_mean_metrics(wd_td_ys_bytag, train_wd_predictions_by_code) test_wd_metrics = ResultsProcessor.compute_mean_metrics(wd_test_ys_bytag, test_wd_predictions_by_code) train_sent_metrics = ResultsProcessor.compute_mean_metrics(sent_test_ys_bycode, test_sent_predictions_by_code) test_sent_metrics = ResultsProcessor.compute_mean_metrics(sent_test_ys_bycode, test_sent_predictions_by_code) with open(out_metrics_file, "w+") as f_metrics_file: s = "" pad = ResultsProcessor.pad_str s += ResultsProcessor.metrics_to_string(train_wd_metrics, test_wd_metrics, "\n%s%s%s" % (pad("TAGGING"), pad("Train"), pad("Test"))) s += ResultsProcessor.metrics_to_string(train_sent_metrics, test_sent_metrics, "\n%s%s%s" % (pad("SENTENCE"), pad("Train"), pad("Test"))) f_metrics_file.write(s) write_categories(out_predictions_file, "CB", out_categories_file) print s
test_decision_functions_by_code = test_classifier_per_code(sent_test_xs, tag2sent_classifier, sent_output_train_test_tags, predict_fn=decision_function_for_tag) else: test_decision_functions_by_code = test_classifier_per_code(sent_test_xs, tag2sent_classifier, sent_output_train_test_tags, predict_fn=probability_for_tag) """ Write out the predicted classes """ with open(out_predictions_file, "w+") as f_output_file: f_output_file.write("Essay|Sent Number|Processed Sentence|Concept Codes|Predictions\n") predictions_to_file(f_output_file, sent_test_ys_bycode, test_sent_predictions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags) with open(out_predicted_margins_file, "w+") as f_output_file: f_output_file.write("Essay|Sent Number|Processed Sentence|Concept Codes|Predicted Confidence\n") predictions_to_file(f_output_file, sent_test_ys_bycode, test_decision_functions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags, output_confidence=True) """ Write out the accuracy metrics """ train_wd_metrics = ResultsProcessor.compute_mean_metrics(wd_td_ys_bytag, train_wd_predictions_by_code) test_wd_metrics = ResultsProcessor.compute_mean_metrics(wd_test_ys_bytag, test_wd_predictions_by_code) train_sent_metrics = ResultsProcessor.compute_mean_metrics(sent_test_ys_bycode, test_sent_predictions_by_code) test_sent_metrics = ResultsProcessor.compute_mean_metrics(sent_test_ys_bycode, test_sent_predictions_by_code) with open(out_metrics_file, "w+") as f_metrics_file: s = "" pad = ResultsProcessor.pad_str s += ResultsProcessor.metrics_to_string(train_wd_metrics, test_wd_metrics, "\n%s%s%s" % (pad("TAGGING"), pad("Train"), pad("Test"))) s += ResultsProcessor.metrics_to_string(train_sent_metrics, test_sent_metrics, "\n%s%s%s" % (pad("SENTENCE"), pad("Train"), pad("Test"))) f_metrics_file.write(s) print s #TODO - need to add logic here for GW #write_categories(out_predictions_file, "CB", out_categories_file)
def evaluate_on_test_set(test_essay_feats, out_predictions_file, out_predicted_margins_file, out_metrics_file, out_categories_file): test_feats, test_tags = flatten_to_wordlevel_feat_tags(test_essay_feats) wd_test_ys_bytag = get_wordlevel_ys_by_code(test_tags, wd_train_tags) test_x = feature_transformer.transform(test_feats) """ TEST Tagger """ test_wd_predictions_by_code = test_classifier_per_code( test_x, tag2word_classifier, wd_test_tags) print "\nRunning Sentence Model" """ SENTENCE LEVEL PREDICTIONS FROM STACKING """ sent_test_xs, sent_test_ys_bycode = get_sent_feature_for_stacking_from_tagging_model( sent_input_feat_tags, sent_input_interaction_tags, test_essay_feats, test_x, wd_test_ys_bytag, tag2word_classifier, SPARSE_SENT_FEATS, LOOK_BACK) """ Test Stack Classifier """ test_sent_predictions_by_code \ = test_classifier_per_code(sent_test_xs, tag2sent_classifier, sent_output_train_test_tags) if USE_SVM: test_decision_functions_by_code = test_classifier_per_code( sent_test_xs, tag2sent_classifier, sent_output_train_test_tags, predict_fn=decision_function_for_tag) else: test_decision_functions_by_code = test_classifier_per_code( sent_test_xs, tag2sent_classifier, sent_output_train_test_tags, predict_fn=probability_for_tag) """ Write out the predicted classes """ with open(out_predictions_file, "w+") as f_output_file: f_output_file.write( "Essay|Sent Number|Processed Sentence|Concept Codes|Predictions\n") predictions_to_file(f_output_file, sent_test_ys_bycode, test_sent_predictions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags) with open(out_predicted_margins_file, "w+") as f_output_file: f_output_file.write( "Essay|Sent Number|Processed Sentence|Concept Codes|Predicted Confidence\n" ) predictions_to_file(f_output_file, sent_test_ys_bycode, test_decision_functions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags, output_confidence=True) """ Write out the accuracy metrics """ train_wd_metrics = ResultsProcessor.compute_mean_metrics( wd_td_ys_bytag, train_wd_predictions_by_code) test_wd_metrics = ResultsProcessor.compute_mean_metrics( wd_test_ys_bytag, test_wd_predictions_by_code) train_sent_metrics = ResultsProcessor.compute_mean_metrics( sent_test_ys_bycode, test_sent_predictions_by_code) test_sent_metrics = ResultsProcessor.compute_mean_metrics( sent_test_ys_bycode, test_sent_predictions_by_code) with open(out_metrics_file, "w+") as f_metrics_file: s = "" pad = ResultsProcessor.pad_str s += ResultsProcessor.metrics_to_string( train_wd_metrics, test_wd_metrics, "\n%s%s%s" % (pad("TAGGING"), pad("Train"), pad("Test"))) s += ResultsProcessor.metrics_to_string( train_sent_metrics, test_sent_metrics, "\n%s%s%s" % (pad("SENTENCE"), pad("Train"), pad("Test"))) f_metrics_file.write(s) write_categories(out_predictions_file, "CB", out_categories_file) print s
) predictions_to_file(f_output_file, sent_test_ys_bycode, test_decision_functions_by_code, test_essay_feats, regular_tags + sent_output_train_test_tags, output_confidence=True) """ Write out the accuracy metrics """ train_wd_metrics = ResultsProcessor.compute_mean_metrics( wd_td_ys_bytag, train_wd_predictions_by_code) test_wd_metrics = ResultsProcessor.compute_mean_metrics( wd_test_ys_bytag, test_wd_predictions_by_code) train_sent_metrics = ResultsProcessor.compute_mean_metrics( sent_test_ys_bycode, test_sent_predictions_by_code) test_sent_metrics = ResultsProcessor.compute_mean_metrics( sent_test_ys_bycode, test_sent_predictions_by_code) with open(out_metrics_file, "w+") as f_metrics_file: s = "" pad = ResultsProcessor.pad_str s += ResultsProcessor.metrics_to_string( train_wd_metrics, test_wd_metrics, "\n%s%s%s" % (pad("TAGGING"), pad("Train"), pad("Test"))) s += ResultsProcessor.metrics_to_string( train_sent_metrics, test_sent_metrics, "\n%s%s%s" % (pad("SENTENCE"), pad("Train"), pad("Test"))) f_metrics_file.write(s) write_categories(out_predictions_file, "CB", out_categories_file) print s