def find_best_threshold(tree, method, input_file, output_file, n=4, idf_enabled=False): if method in [knn_classifier, knn_classifier_xv]: features = get_features(input_file, idf_enabled) write_features("tmp.tab", features) results = knn_classifier(None, outfile="tmp.tab") else: results = method(tree, output=output_file, n=n, idf_enabled=idf_enabled) reference = parse_reference(input_file) # some speedup, read once best_threshold = 0.01 best_accuracy = 0 threshold = 0.01 while(threshold <= 1): threshold = round(threshold,2) classification = classify_results(results, threshold) #write(classification, output_file) #find accuracy acc = evaluate(input_file, output_file, pred_id2label=classification, ref_id2label=reference) print "th:", threshold, "acc:",acc if acc >= best_accuracy: best_threshold = threshold best_accuracy = acc threshold += 0.01 print "best threshold was %.2f with %.4f accuracy" % (best_threshold, best_accuracy)
def main(tree, output, method, threshold, find_best, n=4, idf_enabled=False): #load xml and idf if method in ["word", "lemma", "bleu"]: print "Loading xmlfile" tree = (load_xml.get_pairs(tree), tree) print "done." if idf_enabled: generate_idf_score(tree[0]) elif method in ["print_ted", "ted"]: print "Loading xmlfile" tree = (create_tree.generate_syntax_tree(tree), tree) print "done." if idf_enabled: generate_idf_score(load_xml.get_pairs(tree[1])) elif method in ["features"]: features = get_features(tree, idf_enabled) write_features(output, features) return elif method in ["knn", "knn-xv"]: tree = (tree, tree) #run methods if find_best: find_best_threshold(tree[0], METHODS[method], tree[1], output, n=n, idf_enabled=idf_enabled) else: if method in ["knn", "knn-xv"]: features = get_features(tree[0], idf_enabled=idf_enabled) write_features("features.tab", features) results = METHODS[method](None, outfile="features.tab") else: results = METHODS[method](tree[0], n=n, idf_enabled=idf_enabled, output=output) if method == "print_ted": return classification = classify_results(results, threshold) print "writing output" write(classification, output) print "Accuracy = %.4f" % evaluate(tree[1], output)