def cross_validation(num_folds, featuresets):
subset_size = int(len(featuresets) / num_folds)
print(num_folds, "folds, each of size:", subset_size)
gold = []
predicted = []
accuracy_list = []
precision_list = []
recall_list = []
f1_list = []
Recall = []
Precision = []
F1 = []
true_pos = true_neg = false_pos = false_neg = 0
for i in range(num_folds):
test_this_round = featuresets[(i * subset_size):][:subset_size]
train_this_round = featuresets[:(i * subset_size)] + featuresets[(
(i + 1) * subset_size):]
classifier = nltk.NaiveBayesClassifier.train(train_this_round)
goldlist = []
predictedlist = []
for (features, label) in test_this_round:
goldlist.append(label)
gold.append(label)
predictedlist.append(classifier.classify(features))
predicted.append(classifier.classify(features))
accuracy_this_round = nltk.classify.accuracy(classifier,
test_this_round)
print(i, accuracy_this_round)
accuracy_list.append(accuracy_this_round)
labels = list(set(goldlist))
for lab in labels:
TP = FP = FN = TN = 0
for i, val in enumerate(goldlist):
if val == lab and predictedlist[i] == lab: TP += 1
if val == lab and predictedlist[i] != lab: FN += 1
if val != lab and predictedlist[i] == lab: FP += 1
if val != lab and predictedlist[i] != lab: TN += 1
recall = TP / (TP + FP)
precision = TP / (TP + FN)
recall_list.append(recall)
precision_list.append(precision)
f1_list.append(2 * (recall * precision) / (recall + precision))
if lab == "R":
true_pos = true_pos + TP
true_neg = true_neg + TN
false_pos = false_pos + FP
false_neg = false_neg + FN
Recall.append(true_pos / (true_pos + false_pos))
Recall.append(true_neg / (true_neg + false_neg))
Precision.append(true_pos / (true_pos + false_neg))
Precision.append(true_neg / (true_neg + false_pos))
F1.append((2 * Recall[0] * Precision[0]) / (Recall[0] + Precision[0]))
F1.append((2 * Recall[1] * Precision[1]) / (Recall[1] + Precision[1]))
cm = nltk.ConfusionMatrix(gold, predicted)
print("Average accuracy:", sum(accuracy_list) / num_folds)
print(cm.pretty_format(sort_by_count=True, truncate=9))
print("Party\tPrecision\t Recall\t\t F1")
for i, lab in enumerate(labels):
print(lab, "\t","{:7.3f}".format(Precision[i]), "\t", \
"{:7.3f}".format(Recall[i]), "\t", "{:7.3f}".format(F1[i]))
def run_validation(data_path, k, n):
print("\n\t" + str(k) + "-fold cross-validation:\n")
iterations = []
gold_total = []
silver_total = []
start_time = datetime.datetime.now().replace(microsecond=0)
for i in range(n):
validations, gold, silver = validate_accuracy(read_data(data_path), k)
iterations.append(numpy.mean(validations))
gold_total += gold
silver_total += silver
print("\t{0}.\t".format(i + 1), end='')
for v in validations:
print("{0:.2f} ".format(v), end='')
print("\t{0:.0%}".format(numpy.mean(validations)))
end_time = datetime.datetime.now().replace(microsecond=0)
print("\n\tTotal validation time: " + str(end_time - start_time))
print("\n\tAverage accuracy in {0} iterations: {1:.0%}\n".format(
n, numpy.mean(iterations)))
print(classification_report(gold_total, silver_total))
print("Confusion matrix:")
print(nltk.ConfusionMatrix(gold_total, silver_total))
def evaluate(classifier, data, reviews, output_file):
fh = open(output_file, 'w', encoding='utf-8')
# test on the data
accuracy = nltk.classify.accuracy(classifier, data)
fh.write("{0:10s} {1:8.5f}\n\n".format("Accuracy", accuracy))
features_only = [example[0] for example in data]
reference_labels = [example[1] for example in data]
predicted_labels = classifier.classify_many(features_only)
reference_text = [review[0] for review in reviews]
confusion_matrix = nltk.ConfusionMatrix(reference_labels, predicted_labels)
fh.write(str(confusion_matrix))
fh.write('\n\n')
for reference, predicted, text in zip(
reference_labels,
predicted_labels,
reference_text
):
if reference != predicted:
fh.write("{0} {1}\n{2}\n\n".format(reference, predicted, text))
fh.close()
def basic_analyze(fvecs):
# split in to training and test sets
v_train = fvecs[:2500]
v_test = fvecs[2500:]
# train classifier
nb_classifier = nltk.NaiveBayesClassifier.train(v_train)
#classifier = nltk.classify.maxent.train_maxent_classifier_with_gis(v_train);
# me_classifier = nltk.MaxentClassifier.train(v_train, algorithm='iis', trace=0, max_iter=1, min_lldelta=0.5)
# classify and dump results for interpretation
print '\nNaiveBayesClassifier Accuracy %f' % nltk.classify.accuracy(
nb_classifier, v_test)
# print '\nMaxentClassifier Accuracy %f\n' % nltk.classify.accuracy(me_classifier, v_test)
#print classifier.show_most_informative_features(200)
precisions, recalls, f_measures = measure(nb_classifier, v_test, 0.8)
print 'NB_Precisions:', precisions
print 'NB_Recalls:', recalls
print 'NB_F-Measures:', f_measures
# precisions, recalls, f_measures = measure(me_classifier, v_test, 0.8)
# print 'ME_Precisions:', precisions
# print 'ME_Recalls:', recalls
# print 'ME_F-Measures:', f_measures
# build confusion matrix over test set
test_truth = [s for (t, s) in v_test]
test_predict = [nb_classifier.classify(t) for (t, s) in v_test]
print '\nConfusion Matrix'
print nltk.ConfusionMatrix(test_truth, test_predict)
def evaluate(classifier, features_category_tuples, reference_text, data_set_name=None):
# YOUR CODE GOES HERE
# TODO: evaluate your model
# dev_accuracy = nltk.classify.accuracy(classifier, features_category_tuples)
test_accuracy = nltk.classify.accuracy(
classifier, features_category_tuples)
features_only = [example[0] for example in features_category_tuples]
reference_labels = [example[1] for example in features_category_tuples]
predicted_labels = classifier.classify_many(features_only)
confusion_matrix = nltk.ConfusionMatrix(reference_labels, predicted_labels)
# print(confusion_matrix)
# for reference, predicted, text in zip(reference_labels, predicted_labels, reference_text):
# if reference != predicted:
# print("{0} {1}\n{2}\n\n".format(reference, predicted, text))
name_map = {}
name_map["word_features"] = "ngrams"
name_map["word_pos_features"] = "pos"
name_map["word_pos_liwc_features"] = "liwc"
name_map["best"] = "best"
f = open("output-"+name_map[data_set_name], "w")
f.write("The accuracy of {} is: {}\n".format(data_set_name, test_accuracy))
f.write(str(confusion_matrix))
f.write("\n")
return test_accuracy, confusion_matrix
def cross_validation_accuracy(num_folds, featuresets):
subset_size = int(len(featuresets) / num_folds)
print('Each fold size:', subset_size)
accuracy_list = []
# iterate over the folds
for i in range(num_folds):
test_this_round = featuresets[(i * subset_size):][:subset_size]
train_this_round = featuresets[:(i * subset_size)] + featuresets[(
(i + 1) * subset_size):]
# train using train_this_round
classifier = nltk.NaiveBayesClassifier.train(train_this_round)
# evaluate against test_this_round and save accuracy
accuracy_this_round = nltk.classify.accuracy(classifier,
test_this_round)
gold_lst = []
pred_lst = []
for i, (feature, label) in enumerate(test_this_round):
gold_lst.append(label)
pred_lst.append(classifier.classify(feature))
cm = nltk.ConfusionMatrix(gold_lst, pred_lst)
print(i, accuracy_this_round)
measures(gold_lst, pred_lst)
#print('Recall: ',recall(set(gold_lst),set(pred_lst)))
print(
cm.pretty_format(sort_by_count=True,
show_percents=True,
truncate=9))
accuracy_list.append(accuracy_this_round)
# find mean accuracy over all rounds
print('mean accuracy', sum(accuracy_list) / num_folds)
def evaluation(test_set, classifier): referenceSet = collections.defaultdict(set) testSet = collections.defaultdict(set) referenceSet_cm = [] testSet_cm = [] for index, (tweets, actualLabel) in enumerate(test_set): referenceSet[actualLabel].add(index) referenceSet_cm.append(actualLabel) predictedLabel = classifier.classify(tweets) testSet[predictedLabel].add(index) testSet_cm.append(predictedLabel) # Evaluation of the classification # Accuracy is the percentage of the correct classifications of the test_test (fraction of the labelled data) # Recall describes the completeness of the retrieval. It is defined as the portion of the positive examples retrieved by the process versus the # total number of existing positive examples (including the ones not retrieved by the process). # Precision describes the actual accuracy of the retrieval, and is defined as the portion of the positive examples that exist in the total number of # examples retrieved. print 'Accuracy of the classifier: ', nltk.classify.util.accuracy(classifier, test_set) print '\nTraffic precision: ', nltk.metrics.precision(referenceSet['traffic'], testSet['traffic']) print 'Traffic recall: ', nltk.metrics.recall(referenceSet['traffic'], testSet['traffic']) print 'Traffic F-measure: ', nltk.metrics.f_measure(referenceSet['traffic'], testSet['traffic']) print '\nNon-Traffic precision: ', nltk.metrics.precision(referenceSet['nontraffic'], testSet['nontraffic']) print 'Non-Traffic recall: ', nltk.metrics.recall(referenceSet['nontraffic'], testSet['nontraffic']) print 'Non-Traffic F-measure: ', nltk.metrics.f_measure(referenceSet['nontraffic'], testSet['nontraffic']) print "\n" # Find the Confusion Matrix for the test set cm = nltk.ConfusionMatrix(referenceSet_cm, testSet_cm) print cm.pp(sort_by_count=True, show_percents=True, truncate=9) # Show the 10 features with the greatest gain # classifier.show_most_informative_features()
def main():
# get input:
parser = argparse.ArgumentParser()
parser.add_argument("file", type=str,
nargs='+') #to take files from command prompt
args = parser.parse_args()
pos_tag = open(args.file[0], 'r', encoding="utf8") #get predicted results
pos_test_key = open(args.file[1], 'rt',
encoding="utf8") #get actual/gold-std results
key = pos_test_key.read().split()
key_f = [nltk.tag.str2tuple(t) for t in key]
key_fpair = [(word, tag) for (word, tag) in key_f if tag != None]
y_true = [i[1] for i in key_fpair] #get actual tags into list
"""start working on predicted reults"""
readlines = pos_tag.read()
pos_tag.close()
xc = ast.literal_eval(readlines)
y_pred = [i[1] for i in xc] #get predicted tags in file
AS = accuracy_score(y_true, y_pred) #create accuracy score
CM = nltk.ConfusionMatrix(y_true, y_pred) #create confusion matrix
f = open(args.file[2], "w") #get results into .txt
f.write("Accuracy is:{}\n".format(str(AS)))
f.write(str(CM))
f.close()
def select_evaluate(classifier, tuples):
features = [feat[0] for feat in tuples]
real_labels = [feat[1] for feat in tuples]
accuracy = nltk.classify.accuracy(classifier, tuples)
predicted_labels = classifier.classify_many(features)
confusion_matrix = nltk.ConfusionMatrix(real_labels, predicted_labels)
return accuracy, confusion_matrix
def evaluate(classifier, test_set):
def f_measure(precision, recall, alpha):
try:
return 1 / (alpha * (1 / precision) + (1 - alpha) * 1 / recall)
except ZeroDivisionError:
print("Division by 0")
return 1
test = classifier.batch_classify([fs for (fs, l) in test_set])
gold = [l for (fs, l) in test_set]
matrix = nltk.ConfusionMatrix(gold, test)
print("")
print(matrix)
tp = (matrix[True, True])
fn = (matrix[True, False])
fp = (matrix[False, True])
accuracy = nltk.classify.accuracy(classifier, test_set)
precision = tp / (tp + fp)
recall = tp / (tp + fn)
f = f_measure(precision, recall, 0.5)
print("F-measure: {:.2f} - Accuracy: {:.2f} - Precision: {:.2f} - "
"Recall: {:.2f}".format(f, accuracy, precision, recall))
return [
round(f, 2),
round(accuracy, 2),
round(precision, 2),
round(recall, 2)
]
def evaluate(classifier,
features_category_tuples,
reference_text,
feature_set,
data_set_name=None):
cap = ""
if feature_set == "word_features":
cap = "ngrams"
if feature_set == "word_pos_features":
cap = "pos"
if feature_set == "word_pos_liwc_features":
cap = "liwc"
accuracy = nltk.classify.accuracy(classifier, features_category_tuples)
# TODO: evaluate your model
#raise NotImplemented
fout = open("output-" + cap + ".txt", "w+")
ref = [pair[1] for pair in features_category_tuples]
pred = classifier.classify_many(
[pair[0] for pair in features_category_tuples])
confusion_matrix = nltk.ConfusionMatrix(ref, pred)
i = 0
for t in reference_text:
p = pred[i]
fout.write(p + " " + t + "\n")
i += 1
fout.write(str(accuracy) + "\n" + str(confusion_matrix) + " \n")
fout.close()
return accuracy, confusion_matrix
def evaluate(classifier, data, k, verbose_errors=False):
pos_words = set([str(x) for x in json.load(open('positive_words.json'))])
accuracies, ref, test = [], [], []
print 'Using %s with k = %s' % (classifier, k)
i, bar = 0, pbar(k)
bar.start()
for training, validation, val_sents in k_fold_cross_validation(data, k):
model = classifier.train(training)
# model.show_most_informative_features(50)
accuracies.append(nltk.classify.accuracy(model, validation))
for j, (feat, tag) in enumerate(validation):
guess = model.classify(feat)
ref.append(tag)
test.append(guess)
if guess != tag and verbose_errors:
if guess == 1 and tag == -1:
if any([
word.lower() in pos_words
for word in val_sents[j].split(' ')
]):
print 'guess:', guess, 'actual:', tag, 'SENT:', val_sents[
j]
i += 1
bar.update(i)
bar.finish()
print 'Accuracy: %s' % (sum(accuracies) * 1.0 / len(accuracies))
print nltk.ConfusionMatrix(ref, test)
return model
def compare(gold_standard, comp, gold_links, dev_links):
# # Do not need it actually, but it can be used to get information
# true_positives = nltk.Counter()
# false_negatives = nltk.Counter()
# false_positives = nltk.Counter()
#
# for i in labels:
# for j in labels:
# if i == j:
# true_positives[i] += cm[i, j]
# else:
# false_negatives[i] += cm[i, j]
# false_positives[j] += cm[i, j]
#
# print("true_positives:", true_positives)
# print("false_negatives:", false_negatives)
# print("false_positives:", false_positives)
conf_matrix = nltk.ConfusionMatrix(gold_standard, comp)
classification_rep = classification_report(gold_standard, comp,
list(set(gold_standard + comp)))
correct_num = 0
count_all = 0
for i in range(len(gold_standard)):
count_all += 1
if gold_links[i] == dev_links[i]:
correct_num += 1
links_accuracy = correct_num / count_all # len(gold_standard)
return conf_matrix, classification_rep, links_accuracy
def evaluate(classifier,
features_category_tuples,
reference_text,
outfile,
data_set_name=None):
### YOUR CODE GOES HERE
# TODO: evaluate your model
accuracy = nltk.classify.accuracy(classifier, features_category_tuples)
features = []
classes = []
guesses = []
for f in features_category_tuples:
features.append(f[0])
classes.append(f[1])
guesses.append(classifier.classify(f[0]))
probability = classifier.prob_classify_many(features)
confusion_matrix = nltk.ConfusionMatrix(classes, guesses)
with open(outfile, "w", encoding="utf-8") as fout:
with redirect_stdout(fout):
print(accuracy)
for pdist in probability:
print('%.4f %.4f' %
(pdist.prob('positive'), pdist.prob('negative')))
print(confusion_matrix)
return accuracy, probability, confusion_matrix
def confusion_matrix(real_vals, classifier):
pred_vals = classifier.classify_many([val[0] for val in real_vals])
cm = nltk.ConfusionMatrix([val[1] for val in real_vals], pred_vals)
print("\nConfusion Matrix:\n")
print(
cm.pretty_format(sort_by_count=True,
show_percents=True,
truncate=len(real_vals[0][0])))
def create_confussion_matrix(v_test,classifier):
test_truth = [s for (t,s) in v_test]
test_predict = [classifier.classify(extract_features(t.split())) for (t,s) in v_test]
confussion_matrix=nltk.ConfusionMatrix(test_truth, test_predict )
print 'Confusion Matrix'
print(confussion_matrix)
accuracy=(float)(confussion_matrix._correct)/confussion_matrix._total
print("Accuracy:",accuracy)
def test(self, path):
tagged_sents, sents, tagged_words, words = self.__preprocessing(path)
test_tags = [
tag for sent in sents
for (word, tag) in self.currentTagger.tag(sent)
]
gold_tags = [tag for (word, tag) in tagged_words]
print(nltk.ConfusionMatrix(gold_tags, test_tags))
print("accuracy eval: ", self.currentTagger.evaluate(tagged_sents))
def evaluateMultilabel(result, data, label_index):
data = [l[label_index] for (f, l, i) in data]
cm = nltk.ConfusionMatrix(result, data)
print(cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9))
print_accuracy_per_label(cm, data)
print_macro_f1_per_label(cm, data)
def evaluate_classifier(classifier, test_data):
'''tests the accuracy of the classifier on the test data
it also prints a confusion matrix
'''
gold_st = [t[1] for t in test_data]
test_st = [classifier.classify(t[0]) for t in test_data]
cm = nltk.ConfusionMatrix(gold_st, test_st)
print(cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9))
print('Accuracy: ' + str(nltk.classify.accuracy(classifier, test_data)) +
'\n')
def tagging_confusion_matrix():
train_sents, test_sents = split_dataset()
t0 = nltk.DefaultTagger('NN')
t1 = nltk.UnigramTagger(train_sents, backoff=t0)
t2 = nltk.BigramTagger(train_sents, backoff=t1)
test_tags = [tag for sent in brown.sents(categories='editorial')
for (word, tag) in t2.tag(sent)]
gold_tags = [tag for (word, tag) in brown.tagged_words(categories='editorial')]
print(nltk.ConfusionMatrix(gold_tags, test_tags))
def evaluate(classifier, dataset, binning):
feature_category_tuples, texts = build_features(dataset, "word_features",
binning)
features = [feat[0] for feat in feature_category_tuples]
real_labels = [feat[1] for feat in feature_category_tuples]
accuracy = nltk.classify.accuracy(classifier, feature_category_tuples)
predicted_labels = classifier.classify_many(features)
confusion_matrix = nltk.ConfusionMatrix(real_labels, predicted_labels)
return accuracy, confusion_matrix
def evaluate(classifier, features_category_tuples, reference_text, data_set_name=None):
### YOUR CODE GOES HERE
# TODO: evaluate your model
features = [feat[0] for feat in features_category_tuples]
real_labels = [feat[1] for feat in features_category_tuples]
accuracy = nltk.classify.accuracy(classifier, [(features[i], real_labels[i]) for i in range(len(features))])
predicted_labels = classifier.classify_many(features)
confusion_matrix = nltk.ConfusionMatrix(real_labels, predicted_labels)
return accuracy, confusion_matrix
def cross_validation_PRF(num_folds, featuresets, labels):
subset_size = int(len(featuresets)/num_folds)
print("Each fold size:", subset_size)
print("Test Percentage: ", round(subset_size/len(featuresets),2))
print("Train Percentage: ", round((1 - subset_size/len(featuresets)),2), "\n")
# for the number of labels - start the totals lists with zeroes
num_labels = len(labels)
total_precision_list = [0] * num_labels
total_recall_list = [0] * num_labels
total_F1_list = [0] * num_labels
# iterate over the folds
for i in range(num_folds):
test_this_round = featuresets[(i * subset_size): ][ :subset_size]
train_this_round = featuresets[ :(i * subset_size)] + featuresets[((i + 1) * subset_size): ]
# train using train_this_round
classifier = nltk.NaiveBayesClassifier.train(train_this_round)
accuracy_this_round = nltk.classify.accuracy(classifier, test_this_round)
# evaluate agains the test_this_round to produce
# the gold and predicted labels
goldlist = []
predictedlist = []
for (features, label) in test_this_round:
goldlist.append(label)
predictedlist.append(classifier.classify(features))
# computes evaluation measures for this fold and returns list
# of measures for each label
print("Fold", i, " - Accuracy:", round(accuracy_this_round, 3)* 100, "%\n")
(precision_list, recall_list, F1_list) = eval_measures(goldlist, predictedlist, labels)
# confusion matrix for each fold
cm = nltk.ConfusionMatrix(goldlist, predictedlist)
# confusion matrix with counts
#print(cm.pretty_format(sort_by_count = True, show_percents = False, truncate = 9))
# confusion matrix with percents
print(cm.pretty_format(sort_by_count = True, show_percents = True, truncate = 9))
# take off the triple string to print precision, recall, and F1 for each fold
print('\tPrecision\tRecall\t\tF1')
# print measures for each label
for i, lab in enumerate(labels):
print(lab, '\t', "{:10.3f}".format(precision_list[i]), \
"{:10.3f}".format(recall_list[i]), "{:10.3f}".format(F1_list[i]))
# for each label add to the sums in the total list
for i in range(num_labels):
# for each label, add the 3 measures to the 3 lists of totals
total_precision_list[i] += precision_list[i]
total_recall_list[i] += recall_list[i]
total_F1_list[i] += F1_list[i]
def confusion_matrix(test_data, classifier):
correct_entailment_type = []
predicted_entailment_type = []
for test_case in test_data:
correct_entailment_type.append(test_case[1])
result = classifier.classify(test_case[0])
predicted_entailment_type.append(result)
cm = nltk.ConfusionMatrix(correct_entailment_type,
predicted_entailment_type)
return cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9)
def test3():
def tag_list(tagged_sents):
return [tag for sent in tagged_sents for (word, tag) in sent]
def apply_tagger(tagger, corpus):
return [tagger.tag(nltk.tag.untag(sent)) for sent in corpus]
gold = tag_list(brown.tagged_sents(categories='editorial'))
test = tag_list(
apply_tagger(t2, brown.tagged_sents(categories='editorial')))
cm = nltk.ConfusionMatrix(gold, test)
print(cm.pp(sort_by_count=True, show_precents=True, truncate=9))
def runTests(transitionDict, emissionDict, StagsArray, algorithm):
if algorithm == BIGRAM_HMM:
print("")
print("------------>HMM bigram model for POS<-------------")
if algorithm == ADD_SMOOTH:
print("")
print("------------>HMM Add-one smoothing for POS<-------------")
if algorithm == PSEUDOWORDS:
print("")
print("------------>HMM Pseudo-Words for POS<-------------")
if algorithm == PSEUDOWORDSANDSMOOTH:
print("")
print(
"------------>HMM Pseudo-Words and SMOOTHING for POS<-------------"
)
sumAllTestsErRatesKnown = 0
sumAllTestsErRatesUnkown = 0
sumAllTestsErRatesTotal = 0
count = 0
labelList = []
predictedList = []
for testSentence in testCorpus:
count += 1
sentence = [word for word, tag in testSentence]
tagList = viterbiAlg(sentence, emissionDict, transitionDict,
StagsArray, algorithm)
predictedList.extend(tagList)
labelList.extend([tag for word, tag in testSentence])
erKnown, erUnKnown, totErate = ErrorRateForTest(testSentence, tagList)
sumAllTestsErRatesKnown += erKnown
sumAllTestsErRatesUnkown += erUnKnown
sumAllTestsErRatesTotal += totErate
sumAllTestsErRatesKnown = sumAllTestsErRatesKnown / count
sumAllTestsErRatesUnkown = sumAllTestsErRatesUnkown / count
sumAllTestsErRatesTotal = sumAllTestsErRatesTotal / count
if (algorithm == PSEUDOWORDSANDSMOOTH):
print("confusion Matrix for - ", algorithm)
print("")
confusionMatrix = nltk.ConfusionMatrix(labelList, predictedList)
print(
confusionMatrix.pretty_format(sort_by_count=True,
show_percents=False))
print("-errorRateTest - Known : ", sumAllTestsErRatesKnown)
print("-errorRateTest - UnKnown : ", sumAllTestsErRatesUnkown)
print("-errorRateTest - total : ", sumAllTestsErRatesTotal)
print("")
print("--------------------------------------------")
def create_confusion_matrix(corpus, category, tagger):
# get a list of the gold standard tags, and the tags set by the tagger.
gold = tag_list(corpus.tagged_sents(categories=category))
test = tag_list(
apply_tagger(tagger, corpus.tagged_sents(categories=category)))
# create the confusion matrix and return it in a pretty-printed format.
cm = nltk.ConfusionMatrix(gold, test)
return cm.pretty_format(sort_by_count=True,
show_percents=True,
truncate=20)
def run(self):
args = self.get_arguments()
model_file = args[0]
filenames = args[1:]
if not os.path.exists(model_file):
print("ERROR: model file '%s' doesn't exist" % model_file)
return True
model = SentimentModel.load(model_file)
evaluator = ModelEvaluator(model, filenames)
gold, test = evaluator.get_gold_comparison()
print("Confusion matrix:")
cm = nltk.ConfusionMatrix(gold, test)
print(cm.pp(sort_by_count=True, show_percents=True))
summary = evaluator.evaluate()
precision_sum, recall_sum, fmeasure_sum = 0, 0, 0
print(" TP TN FP FN precision recall f-measure")
rowstr = "%(label)s %(tp)6s %(tn)6s %(fp)6s %(fn)6s %(precision)2.5f %(recall)2.5f %(fmeasure)2.5f"
for sentiment, label in TEXT_LABELS.items():
print(rowstr % {
"label": '%s:' % label.rjust(10),
"tp": summary[sentiment]["true-positives"],
"fp": summary[sentiment]["false-positives"],
"tn": summary[sentiment]["true-negatives"],
"fn": summary[sentiment]["false-negatives"],
"precision": summary[sentiment]["precision"],
"recall": summary[sentiment]["recall"],
"fmeasure": summary[sentiment]["f-measure"],
})
precision_sum += summary[sentiment]["precision"]
recall_sum += summary[sentiment]["recall"]
fmeasure_sum += summary[sentiment]["f-measure"]
precision_avg = precision_sum / len(TEXT_LABELS)
recall_avg = recall_sum / len(TEXT_LABELS)
fmeasure_avg = fmeasure_sum / len(TEXT_LABELS)
print(rowstr % {
"label": "average:".rjust(11),
"tp": "", "fp": "", "tn": "", "fn": "",
"precision": precision_avg,
"recall": recall_avg,
"fmeasure": fmeasure_avg,
})
return False
def classifierConfusionMatrix(self, posTokenizedPostsTest,
negTokenizedPostsTest, posLIWCPosts,
negLIWCPosts):
gold = self.getFeatureSetForAllPosts(posTokenizedPostsTest,
negTokenizedPostsTest,
posLIWCPosts, negLIWCPosts)
feature_set = self.removeClassesFromFeatures(gold)
predictions = self.classifier.classify_many(feature_set)
gs = []
for i in range(0, len(gold)):
g, v = gold[i]
gs += [v]
return nltk.ConfusionMatrix(gs, predictions)
def calculateStats(self, i):
'''Function to calculate all stats'''
#calculate cm for this iteration
ref = []
tagged = []
for f, e in self.testB:
ref.append(self.classifierB.classify(f))
tagged.append(e)
self.cm.append(nltk.ConfusionMatrix(ref, tagged))
#self.informativeFeatures.append(self.classifierB.most_informative_features(10))
print()
#calculate precision and recall for this iteration for each category
refsets = defaultdict(set)
testsets = defaultdict(set)
#allCount = 0
#noEventCount = 0
for n, (feats, label) in enumerate(self.testB):
#allCount += 1
#if label == "geen_event":
# noEventCount += 1
refsets[label].add(n)
observed = self.classifierB.classify(feats)
# uncomment voor wel_event
#if label != "geen_event":
# refsets["wel_event"].add(n)
#if observed != "geen_event":
# testsets["wel_event"].add(n)
testsets[observed].add(n)
#print("Accuracy geen_event (baseline) is", noEventCount/allCount) #
self.accuracy.append(
nltk.classify.accuracy(self.classifierB, self.testB))
#for elke category precision and recall berekenen.
for category in self.categories:
if category in testsets:
self.result[category]["p"].append(
nltk.metrics.precision(refsets[category],
testsets[category]))
self.result[category]["r"].append(
nltk.metrics.recall(refsets[category], testsets[category]))
self.result[category]["f"].append(
nltk.metrics.f_measure(refsets[category],
testsets[category]))
else:
self.result[category]["p"].append(float(0))
self.result[category]["r"].append(float(0))
self.result[category]["f"].append(float(0))
