def Evaluation_lst(gold_label, predict_label, print_all=False): binary_alphabet = Alphabet() for i in range(20): binary_alphabet.add(DICT_INDEX_TO_LABEL[i]) cm = ConfusionMatrix(binary_alphabet) cm.add_list(predict_label, gold_label) if print_all: cm.print_out() overall_accuracy = cm.get_accuracy() return overall_accuracy
def Evalation_list(gold_label, predict_label, print_all=False): binary_alphabet = Alphabet() for i in range(2): binary_alphabet.add(str(i)) cm = ConfusionMatrix(binary_alphabet) predict_label = list(map(str, predict_label)) gold_label = list(map(str, gold_label)) cm.add_list(predict_label, gold_label) if print_all: cm.print_out() overall_accuracy = cm.get_accuracy() return overall_accuracy
def Evaluation(gold_file_path, predict_file_path): with open(gold_file_path) as gold_file, open(predict_file_path) as predict_file: gold_list = [ line.strip().split('\t')[0] for line in gold_file] predicted_list = [line.strip().split("\t#\t")[0] for line in predict_file] binary_alphabet = Alphabet() for i in range(18): binary_alphabet.add(DICT_INDEX_TO_LABEL[i]) cm = ConfusionMatrix(binary_alphabet) cm.add_list(predicted_list, gold_list) cm.print_out() macro_p, macro_r, macro_f1 = cm.get_average_prf() overall_accuracy = cm.get_accuracy() return overall_accuracy, macro_p, macro_r, macro_f1
def evaluate(gold_file, pred_file): with codecs.open(gold_file, encoding="utf-8") as fin_gold, codecs.open( pred_file, encoding="utf-8") as fin_pred: dict_P_to_url_label = {} for line in fin_gold: P, url, label, _ = line.strip().split("\t") if P not in dict_P_to_url_label: dict_P_to_url_label[P] = set() dict_P_to_url_label[P].add((url.strip(), label)) # predict_set = set() for line in fin_pred: url, s, p, o, confidence = line.strip().split("\t") predict_set.add((url.strip(), p)) alphabet = Alphabet() alphabet.add("0") alphabet.add("1") # 评估 marco_p, marco_r, marco_f = 0, 0, 0 N = 0 for P in sorted(dict_P_to_url_label.keys()): confusionMatrix = ConfusionMatrix(alphabet) recall_error_cases = [] precision_error_cases = [] for url, label in dict_P_to_url_label[P]: pred = "0" if (url, P) in predict_set: pred = "1" if label != pred: if label == "1" and pred == "0": recall_error_cases.append("%s\t%s->%s" % (url, label, pred)) if label == "0" and pred == "1": precision_error_cases.append("%s\t%s->%s" % (url, label, pred)) confusionMatrix.add(pred, label) print "==" * 40 print P print confusionMatrix.print_out() p, r, f = confusionMatrix.get_prf("1") marco_p += p marco_r += r marco_f += f N += 1 print "\n==>recall error cases:" print "\n".join(recall_error_cases) print "\n==>precision error cases:" print "\n".join(precision_error_cases) print "**" * 40 print "marco, P: %f; R: %f; F1: %f" % (marco_p / N, marco_r / N, marco_f / N)
def evaluate(gold_file, pred_file): with codecs.open(gold_file, encoding="utf-8") as fin_gold, codecs.open(pred_file, encoding="utf-8") as fin_pred: dict_P_to_url_label = {} for line in fin_gold: P, url, label, _ = line.strip().split("\t") if P not in dict_P_to_url_label: dict_P_to_url_label[P] = set() dict_P_to_url_label[P].add((url.strip(), label)) # predict_set = set() for line in fin_pred: url, s, p, o, confidence = line.strip().split("\t") predict_set.add((url.strip(), p)) alphabet = Alphabet() alphabet.add("0") alphabet.add("1") # 评估 marco_p, marco_r, marco_f = 0, 0, 0 N = 0 for P in sorted(dict_P_to_url_label.keys()): confusionMatrix = ConfusionMatrix(alphabet) recall_error_cases = [] precision_error_cases= [] for url, label in dict_P_to_url_label[P]: pred = "0" if (url, P) in predict_set: pred = "1" if label != pred: if label == "1" and pred == "0": recall_error_cases.append("%s\t%s->%s" % (url, label, pred)) if label == "0" and pred == "1": precision_error_cases.append("%s\t%s->%s" % (url, label, pred)) confusionMatrix.add(pred, label) print "==" * 40 print P print confusionMatrix.print_out() p, r, f = confusionMatrix.get_prf("1") marco_p += p marco_r += r marco_f += f N += 1 print "\n==>recall error cases:" print "\n".join(recall_error_cases) print "\n==>precision error cases:" print "\n".join(precision_error_cases) print "**" * 40 print "marco, P: %f; R: %f; F1: %f" % (marco_p / N, marco_r / N, marco_f / N)