def Evaluation_lst(gold_label, predict_label, print_all=False):
binary_alphabet = Alphabet()
for i in range(20):
binary_alphabet.add(DICT_INDEX_TO_LABEL[i])
cm = ConfusionMatrix(binary_alphabet)
cm.add_list(predict_label, gold_label)
if print_all:
cm.print_out()
overall_accuracy = cm.get_accuracy()
return overall_accuracy
def Evalation_list(gold_label, predict_label, print_all=False):
binary_alphabet = Alphabet()
for i in range(2):
binary_alphabet.add(str(i))
cm = ConfusionMatrix(binary_alphabet)
predict_label = list(map(str, predict_label))
gold_label = list(map(str, gold_label))
cm.add_list(predict_label, gold_label)
if print_all:
cm.print_out()
overall_accuracy = cm.get_accuracy()
return overall_accuracy
def Evaluation(gold_file_path, predict_file_path):
with open(gold_file_path) as gold_file, open(predict_file_path) as predict_file:
gold_list = [ line.strip().split('\t')[0] for line in gold_file]
predicted_list = [line.strip().split("\t#\t")[0] for line in predict_file]
binary_alphabet = Alphabet()
for i in range(18):
binary_alphabet.add(DICT_INDEX_TO_LABEL[i])
cm = ConfusionMatrix(binary_alphabet)
cm.add_list(predicted_list, gold_list)
cm.print_out()
macro_p, macro_r, macro_f1 = cm.get_average_prf()
overall_accuracy = cm.get_accuracy()
return overall_accuracy, macro_p, macro_r, macro_f1
def evaluate(gold_file, pred_file):
with codecs.open(gold_file, encoding="utf-8") as fin_gold, codecs.open(
pred_file, encoding="utf-8") as fin_pred:
dict_P_to_url_label = {}
for line in fin_gold:
P, url, label, _ = line.strip().split("\t")
if P not in dict_P_to_url_label:
dict_P_to_url_label[P] = set()
dict_P_to_url_label[P].add((url.strip(), label))
#
predict_set = set()
for line in fin_pred:
url, s, p, o, confidence = line.strip().split("\t")
predict_set.add((url.strip(), p))
alphabet = Alphabet()
alphabet.add("0")
alphabet.add("1")
# 评估
marco_p, marco_r, marco_f = 0, 0, 0
N = 0
for P in sorted(dict_P_to_url_label.keys()):
confusionMatrix = ConfusionMatrix(alphabet)
recall_error_cases = []
precision_error_cases = []
for url, label in dict_P_to_url_label[P]:
pred = "0"
if (url, P) in predict_set:
pred = "1"
if label != pred:
if label == "1" and pred == "0":
recall_error_cases.append("%s\t%s->%s" %
(url, label, pred))
if label == "0" and pred == "1":
precision_error_cases.append("%s\t%s->%s" %
(url, label, pred))
confusionMatrix.add(pred, label)
print "==" * 40
print P
print
confusionMatrix.print_out()
p, r, f = confusionMatrix.get_prf("1")
marco_p += p
marco_r += r
marco_f += f
N += 1
print "\n==>recall error cases:"
print "\n".join(recall_error_cases)
print "\n==>precision error cases:"
print "\n".join(precision_error_cases)
print "**" * 40
print "marco, P: %f; R: %f; F1: %f" % (marco_p / N, marco_r / N,
marco_f / N)
def evaluate(gold_file, pred_file):
with codecs.open(gold_file, encoding="utf-8") as fin_gold, codecs.open(pred_file, encoding="utf-8") as fin_pred:
dict_P_to_url_label = {}
for line in fin_gold:
P, url, label, _ = line.strip().split("\t")
if P not in dict_P_to_url_label:
dict_P_to_url_label[P] = set()
dict_P_to_url_label[P].add((url.strip(), label))
#
predict_set = set()
for line in fin_pred:
url, s, p, o, confidence = line.strip().split("\t")
predict_set.add((url.strip(), p))
alphabet = Alphabet()
alphabet.add("0")
alphabet.add("1")
# 评估
marco_p, marco_r, marco_f = 0, 0, 0
N = 0
for P in sorted(dict_P_to_url_label.keys()):
confusionMatrix = ConfusionMatrix(alphabet)
recall_error_cases = []
precision_error_cases= []
for url, label in dict_P_to_url_label[P]:
pred = "0"
if (url, P) in predict_set:
pred = "1"
if label != pred:
if label == "1" and pred == "0":
recall_error_cases.append("%s\t%s->%s" % (url, label, pred))
if label == "0" and pred == "1":
precision_error_cases.append("%s\t%s->%s" % (url, label, pred))
confusionMatrix.add(pred, label)
print "==" * 40
print P
print
confusionMatrix.print_out()
p, r, f = confusionMatrix.get_prf("1")
marco_p += p
marco_r += r
marco_f += f
N += 1
print "\n==>recall error cases:"
print "\n".join(recall_error_cases)
print "\n==>precision error cases:"
print "\n".join(precision_error_cases)
print "**" * 40
print "marco, P: %f; R: %f; F1: %f" % (marco_p / N, marco_r / N, marco_f / N)
