def cross_validation(num_folds, featuresets): subset_size = int(len(featuresets) / num_folds) print(num_folds, "folds, each of size:", subset_size) gold = [] predicted = [] accuracy_list = [] precision_list = [] recall_list = [] f1_list = [] Recall = [] Precision = [] F1 = [] true_pos = true_neg = false_pos = false_neg = 0 for i in range(num_folds): test_this_round = featuresets[(i * subset_size):][:subset_size] train_this_round = featuresets[:(i * subset_size)] + featuresets[( (i + 1) * subset_size):] classifier = nltk.NaiveBayesClassifier.train(train_this_round) goldlist = [] predictedlist = [] for (features, label) in test_this_round: goldlist.append(label) gold.append(label) predictedlist.append(classifier.classify(features)) predicted.append(classifier.classify(features)) accuracy_this_round = nltk.classify.accuracy(classifier, test_this_round) print(i, accuracy_this_round) accuracy_list.append(accuracy_this_round) labels = list(set(goldlist)) for lab in labels: TP = FP = FN = TN = 0 for i, val in enumerate(goldlist): if val == lab and predictedlist[i] == lab: TP += 1 if val == lab and predictedlist[i] != lab: FN += 1 if val != lab and predictedlist[i] == lab: FP += 1 if val != lab and predictedlist[i] != lab: TN += 1 recall = TP / (TP + FP) precision = TP / (TP + FN) recall_list.append(recall) precision_list.append(precision) f1_list.append(2 * (recall * precision) / (recall + precision)) if lab == "R": true_pos = true_pos + TP true_neg = true_neg + TN false_pos = false_pos + FP false_neg = false_neg + FN Recall.append(true_pos / (true_pos + false_pos)) Recall.append(true_neg / (true_neg + false_neg)) Precision.append(true_pos / (true_pos + false_neg)) Precision.append(true_neg / (true_neg + false_pos)) F1.append((2 * Recall[0] * Precision[0]) / (Recall[0] + Precision[0])) F1.append((2 * Recall[1] * Precision[1]) / (Recall[1] + Precision[1])) cm = nltk.ConfusionMatrix(gold, predicted) print("Average accuracy:", sum(accuracy_list) / num_folds) print(cm.pretty_format(sort_by_count=True, truncate=9)) print("Party\tPrecision\t Recall\t\t F1") for i, lab in enumerate(labels): print(lab, "\t","{:7.3f}".format(Precision[i]), "\t", \ "{:7.3f}".format(Recall[i]), "\t", "{:7.3f}".format(F1[i]))
def run_validation(data_path, k, n): print("\n\t" + str(k) + "-fold cross-validation:\n") iterations = [] gold_total = [] silver_total = [] start_time = datetime.datetime.now().replace(microsecond=0) for i in range(n): validations, gold, silver = validate_accuracy(read_data(data_path), k) iterations.append(numpy.mean(validations)) gold_total += gold silver_total += silver print("\t{0}.\t".format(i + 1), end='') for v in validations: print("{0:.2f} ".format(v), end='') print("\t{0:.0%}".format(numpy.mean(validations))) end_time = datetime.datetime.now().replace(microsecond=0) print("\n\tTotal validation time: " + str(end_time - start_time)) print("\n\tAverage accuracy in {0} iterations: {1:.0%}\n".format( n, numpy.mean(iterations))) print(classification_report(gold_total, silver_total)) print("Confusion matrix:") print(nltk.ConfusionMatrix(gold_total, silver_total))
def evaluate(classifier, data, reviews, output_file): fh = open(output_file, 'w', encoding='utf-8') # test on the data accuracy = nltk.classify.accuracy(classifier, data) fh.write("{0:10s} {1:8.5f}\n\n".format("Accuracy", accuracy)) features_only = [example[0] for example in data] reference_labels = [example[1] for example in data] predicted_labels = classifier.classify_many(features_only) reference_text = [review[0] for review in reviews] confusion_matrix = nltk.ConfusionMatrix(reference_labels, predicted_labels) fh.write(str(confusion_matrix)) fh.write('\n\n') for reference, predicted, text in zip( reference_labels, predicted_labels, reference_text ): if reference != predicted: fh.write("{0} {1}\n{2}\n\n".format(reference, predicted, text)) fh.close()
def basic_analyze(fvecs): # split in to training and test sets v_train = fvecs[:2500] v_test = fvecs[2500:] # train classifier nb_classifier = nltk.NaiveBayesClassifier.train(v_train) #classifier = nltk.classify.maxent.train_maxent_classifier_with_gis(v_train); # me_classifier = nltk.MaxentClassifier.train(v_train, algorithm='iis', trace=0, max_iter=1, min_lldelta=0.5) # classify and dump results for interpretation print '\nNaiveBayesClassifier Accuracy %f' % nltk.classify.accuracy( nb_classifier, v_test) # print '\nMaxentClassifier Accuracy %f\n' % nltk.classify.accuracy(me_classifier, v_test) #print classifier.show_most_informative_features(200) precisions, recalls, f_measures = measure(nb_classifier, v_test, 0.8) print 'NB_Precisions:', precisions print 'NB_Recalls:', recalls print 'NB_F-Measures:', f_measures # precisions, recalls, f_measures = measure(me_classifier, v_test, 0.8) # print 'ME_Precisions:', precisions # print 'ME_Recalls:', recalls # print 'ME_F-Measures:', f_measures # build confusion matrix over test set test_truth = [s for (t, s) in v_test] test_predict = [nb_classifier.classify(t) for (t, s) in v_test] print '\nConfusion Matrix' print nltk.ConfusionMatrix(test_truth, test_predict)
def evaluate(classifier, features_category_tuples, reference_text, data_set_name=None): # YOUR CODE GOES HERE # TODO: evaluate your model # dev_accuracy = nltk.classify.accuracy(classifier, features_category_tuples) test_accuracy = nltk.classify.accuracy( classifier, features_category_tuples) features_only = [example[0] for example in features_category_tuples] reference_labels = [example[1] for example in features_category_tuples] predicted_labels = classifier.classify_many(features_only) confusion_matrix = nltk.ConfusionMatrix(reference_labels, predicted_labels) # print(confusion_matrix) # for reference, predicted, text in zip(reference_labels, predicted_labels, reference_text): # if reference != predicted: # print("{0} {1}\n{2}\n\n".format(reference, predicted, text)) name_map = {} name_map["word_features"] = "ngrams" name_map["word_pos_features"] = "pos" name_map["word_pos_liwc_features"] = "liwc" name_map["best"] = "best" f = open("output-"+name_map[data_set_name], "w") f.write("The accuracy of {} is: {}\n".format(data_set_name, test_accuracy)) f.write(str(confusion_matrix)) f.write("\n") return test_accuracy, confusion_matrix
def cross_validation_accuracy(num_folds, featuresets): subset_size = int(len(featuresets) / num_folds) print('Each fold size:', subset_size) accuracy_list = [] # iterate over the folds for i in range(num_folds): test_this_round = featuresets[(i * subset_size):][:subset_size] train_this_round = featuresets[:(i * subset_size)] + featuresets[( (i + 1) * subset_size):] # train using train_this_round classifier = nltk.NaiveBayesClassifier.train(train_this_round) # evaluate against test_this_round and save accuracy accuracy_this_round = nltk.classify.accuracy(classifier, test_this_round) gold_lst = [] pred_lst = [] for i, (feature, label) in enumerate(test_this_round): gold_lst.append(label) pred_lst.append(classifier.classify(feature)) cm = nltk.ConfusionMatrix(gold_lst, pred_lst) print(i, accuracy_this_round) measures(gold_lst, pred_lst) #print('Recall: ',recall(set(gold_lst),set(pred_lst))) print( cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9)) accuracy_list.append(accuracy_this_round) # find mean accuracy over all rounds print('mean accuracy', sum(accuracy_list) / num_folds)
def evaluation(test_set, classifier): referenceSet = collections.defaultdict(set) testSet = collections.defaultdict(set) referenceSet_cm = [] testSet_cm = [] for index, (tweets, actualLabel) in enumerate(test_set): referenceSet[actualLabel].add(index) referenceSet_cm.append(actualLabel) predictedLabel = classifier.classify(tweets) testSet[predictedLabel].add(index) testSet_cm.append(predictedLabel) # Evaluation of the classification # Accuracy is the percentage of the correct classifications of the test_test (fraction of the labelled data) # Recall describes the completeness of the retrieval. It is defined as the portion of the positive examples retrieved by the process versus the # total number of existing positive examples (including the ones not retrieved by the process). # Precision describes the actual accuracy of the retrieval, and is defined as the portion of the positive examples that exist in the total number of # examples retrieved. print 'Accuracy of the classifier: ', nltk.classify.util.accuracy(classifier, test_set) print '\nTraffic precision: ', nltk.metrics.precision(referenceSet['traffic'], testSet['traffic']) print 'Traffic recall: ', nltk.metrics.recall(referenceSet['traffic'], testSet['traffic']) print 'Traffic F-measure: ', nltk.metrics.f_measure(referenceSet['traffic'], testSet['traffic']) print '\nNon-Traffic precision: ', nltk.metrics.precision(referenceSet['nontraffic'], testSet['nontraffic']) print 'Non-Traffic recall: ', nltk.metrics.recall(referenceSet['nontraffic'], testSet['nontraffic']) print 'Non-Traffic F-measure: ', nltk.metrics.f_measure(referenceSet['nontraffic'], testSet['nontraffic']) print "\n" # Find the Confusion Matrix for the test set cm = nltk.ConfusionMatrix(referenceSet_cm, testSet_cm) print cm.pp(sort_by_count=True, show_percents=True, truncate=9) # Show the 10 features with the greatest gain # classifier.show_most_informative_features()
def main(): # get input: parser = argparse.ArgumentParser() parser.add_argument("file", type=str, nargs='+') #to take files from command prompt args = parser.parse_args() pos_tag = open(args.file[0], 'r', encoding="utf8") #get predicted results pos_test_key = open(args.file[1], 'rt', encoding="utf8") #get actual/gold-std results key = pos_test_key.read().split() key_f = [nltk.tag.str2tuple(t) for t in key] key_fpair = [(word, tag) for (word, tag) in key_f if tag != None] y_true = [i[1] for i in key_fpair] #get actual tags into list """start working on predicted reults""" readlines = pos_tag.read() pos_tag.close() xc = ast.literal_eval(readlines) y_pred = [i[1] for i in xc] #get predicted tags in file AS = accuracy_score(y_true, y_pred) #create accuracy score CM = nltk.ConfusionMatrix(y_true, y_pred) #create confusion matrix f = open(args.file[2], "w") #get results into .txt f.write("Accuracy is:{}\n".format(str(AS))) f.write(str(CM)) f.close()
def select_evaluate(classifier, tuples): features = [feat[0] for feat in tuples] real_labels = [feat[1] for feat in tuples] accuracy = nltk.classify.accuracy(classifier, tuples) predicted_labels = classifier.classify_many(features) confusion_matrix = nltk.ConfusionMatrix(real_labels, predicted_labels) return accuracy, confusion_matrix
def evaluate(classifier, test_set): def f_measure(precision, recall, alpha): try: return 1 / (alpha * (1 / precision) + (1 - alpha) * 1 / recall) except ZeroDivisionError: print("Division by 0") return 1 test = classifier.batch_classify([fs for (fs, l) in test_set]) gold = [l for (fs, l) in test_set] matrix = nltk.ConfusionMatrix(gold, test) print("") print(matrix) tp = (matrix[True, True]) fn = (matrix[True, False]) fp = (matrix[False, True]) accuracy = nltk.classify.accuracy(classifier, test_set) precision = tp / (tp + fp) recall = tp / (tp + fn) f = f_measure(precision, recall, 0.5) print("F-measure: {:.2f} - Accuracy: {:.2f} - Precision: {:.2f} - " "Recall: {:.2f}".format(f, accuracy, precision, recall)) return [ round(f, 2), round(accuracy, 2), round(precision, 2), round(recall, 2) ]
def evaluate(classifier, features_category_tuples, reference_text, feature_set, data_set_name=None): cap = "" if feature_set == "word_features": cap = "ngrams" if feature_set == "word_pos_features": cap = "pos" if feature_set == "word_pos_liwc_features": cap = "liwc" accuracy = nltk.classify.accuracy(classifier, features_category_tuples) # TODO: evaluate your model #raise NotImplemented fout = open("output-" + cap + ".txt", "w+") ref = [pair[1] for pair in features_category_tuples] pred = classifier.classify_many( [pair[0] for pair in features_category_tuples]) confusion_matrix = nltk.ConfusionMatrix(ref, pred) i = 0 for t in reference_text: p = pred[i] fout.write(p + " " + t + "\n") i += 1 fout.write(str(accuracy) + "\n" + str(confusion_matrix) + " \n") fout.close() return accuracy, confusion_matrix
def evaluate(classifier, data, k, verbose_errors=False): pos_words = set([str(x) for x in json.load(open('positive_words.json'))]) accuracies, ref, test = [], [], [] print 'Using %s with k = %s' % (classifier, k) i, bar = 0, pbar(k) bar.start() for training, validation, val_sents in k_fold_cross_validation(data, k): model = classifier.train(training) # model.show_most_informative_features(50) accuracies.append(nltk.classify.accuracy(model, validation)) for j, (feat, tag) in enumerate(validation): guess = model.classify(feat) ref.append(tag) test.append(guess) if guess != tag and verbose_errors: if guess == 1 and tag == -1: if any([ word.lower() in pos_words for word in val_sents[j].split(' ') ]): print 'guess:', guess, 'actual:', tag, 'SENT:', val_sents[ j] i += 1 bar.update(i) bar.finish() print 'Accuracy: %s' % (sum(accuracies) * 1.0 / len(accuracies)) print nltk.ConfusionMatrix(ref, test) return model
def compare(gold_standard, comp, gold_links, dev_links): # # Do not need it actually, but it can be used to get information # true_positives = nltk.Counter() # false_negatives = nltk.Counter() # false_positives = nltk.Counter() # # for i in labels: # for j in labels: # if i == j: # true_positives[i] += cm[i, j] # else: # false_negatives[i] += cm[i, j] # false_positives[j] += cm[i, j] # # print("true_positives:", true_positives) # print("false_negatives:", false_negatives) # print("false_positives:", false_positives) conf_matrix = nltk.ConfusionMatrix(gold_standard, comp) classification_rep = classification_report(gold_standard, comp, list(set(gold_standard + comp))) correct_num = 0 count_all = 0 for i in range(len(gold_standard)): count_all += 1 if gold_links[i] == dev_links[i]: correct_num += 1 links_accuracy = correct_num / count_all # len(gold_standard) return conf_matrix, classification_rep, links_accuracy
def evaluate(classifier, features_category_tuples, reference_text, outfile, data_set_name=None): ### YOUR CODE GOES HERE # TODO: evaluate your model accuracy = nltk.classify.accuracy(classifier, features_category_tuples) features = [] classes = [] guesses = [] for f in features_category_tuples: features.append(f[0]) classes.append(f[1]) guesses.append(classifier.classify(f[0])) probability = classifier.prob_classify_many(features) confusion_matrix = nltk.ConfusionMatrix(classes, guesses) with open(outfile, "w", encoding="utf-8") as fout: with redirect_stdout(fout): print(accuracy) for pdist in probability: print('%.4f %.4f' % (pdist.prob('positive'), pdist.prob('negative'))) print(confusion_matrix) return accuracy, probability, confusion_matrix
def confusion_matrix(real_vals, classifier): pred_vals = classifier.classify_many([val[0] for val in real_vals]) cm = nltk.ConfusionMatrix([val[1] for val in real_vals], pred_vals) print("\nConfusion Matrix:\n") print( cm.pretty_format(sort_by_count=True, show_percents=True, truncate=len(real_vals[0][0])))
def create_confussion_matrix(v_test,classifier): test_truth = [s for (t,s) in v_test] test_predict = [classifier.classify(extract_features(t.split())) for (t,s) in v_test] confussion_matrix=nltk.ConfusionMatrix(test_truth, test_predict ) print 'Confusion Matrix' print(confussion_matrix) accuracy=(float)(confussion_matrix._correct)/confussion_matrix._total print("Accuracy:",accuracy)
def test(self, path): tagged_sents, sents, tagged_words, words = self.__preprocessing(path) test_tags = [ tag for sent in sents for (word, tag) in self.currentTagger.tag(sent) ] gold_tags = [tag for (word, tag) in tagged_words] print(nltk.ConfusionMatrix(gold_tags, test_tags)) print("accuracy eval: ", self.currentTagger.evaluate(tagged_sents))
def evaluateMultilabel(result, data, label_index): data = [l[label_index] for (f, l, i) in data] cm = nltk.ConfusionMatrix(result, data) print(cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9)) print_accuracy_per_label(cm, data) print_macro_f1_per_label(cm, data)
def evaluate_classifier(classifier, test_data): '''tests the accuracy of the classifier on the test data it also prints a confusion matrix ''' gold_st = [t[1] for t in test_data] test_st = [classifier.classify(t[0]) for t in test_data] cm = nltk.ConfusionMatrix(gold_st, test_st) print(cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9)) print('Accuracy: ' + str(nltk.classify.accuracy(classifier, test_data)) + '\n')
def tagging_confusion_matrix(): train_sents, test_sents = split_dataset() t0 = nltk.DefaultTagger('NN') t1 = nltk.UnigramTagger(train_sents, backoff=t0) t2 = nltk.BigramTagger(train_sents, backoff=t1) test_tags = [tag for sent in brown.sents(categories='editorial') for (word, tag) in t2.tag(sent)] gold_tags = [tag for (word, tag) in brown.tagged_words(categories='editorial')] print(nltk.ConfusionMatrix(gold_tags, test_tags))
def evaluate(classifier, dataset, binning): feature_category_tuples, texts = build_features(dataset, "word_features", binning) features = [feat[0] for feat in feature_category_tuples] real_labels = [feat[1] for feat in feature_category_tuples] accuracy = nltk.classify.accuracy(classifier, feature_category_tuples) predicted_labels = classifier.classify_many(features) confusion_matrix = nltk.ConfusionMatrix(real_labels, predicted_labels) return accuracy, confusion_matrix
def evaluate(classifier, features_category_tuples, reference_text, data_set_name=None): ### YOUR CODE GOES HERE # TODO: evaluate your model features = [feat[0] for feat in features_category_tuples] real_labels = [feat[1] for feat in features_category_tuples] accuracy = nltk.classify.accuracy(classifier, [(features[i], real_labels[i]) for i in range(len(features))]) predicted_labels = classifier.classify_many(features) confusion_matrix = nltk.ConfusionMatrix(real_labels, predicted_labels) return accuracy, confusion_matrix
def cross_validation_PRF(num_folds, featuresets, labels): subset_size = int(len(featuresets)/num_folds) print("Each fold size:", subset_size) print("Test Percentage: ", round(subset_size/len(featuresets),2)) print("Train Percentage: ", round((1 - subset_size/len(featuresets)),2), "\n") # for the number of labels - start the totals lists with zeroes num_labels = len(labels) total_precision_list = [0] * num_labels total_recall_list = [0] * num_labels total_F1_list = [0] * num_labels # iterate over the folds for i in range(num_folds): test_this_round = featuresets[(i * subset_size): ][ :subset_size] train_this_round = featuresets[ :(i * subset_size)] + featuresets[((i + 1) * subset_size): ] # train using train_this_round classifier = nltk.NaiveBayesClassifier.train(train_this_round) accuracy_this_round = nltk.classify.accuracy(classifier, test_this_round) # evaluate agains the test_this_round to produce # the gold and predicted labels goldlist = [] predictedlist = [] for (features, label) in test_this_round: goldlist.append(label) predictedlist.append(classifier.classify(features)) # computes evaluation measures for this fold and returns list # of measures for each label print("Fold", i, " - Accuracy:", round(accuracy_this_round, 3)* 100, "%\n") (precision_list, recall_list, F1_list) = eval_measures(goldlist, predictedlist, labels) # confusion matrix for each fold cm = nltk.ConfusionMatrix(goldlist, predictedlist) # confusion matrix with counts #print(cm.pretty_format(sort_by_count = True, show_percents = False, truncate = 9)) # confusion matrix with percents print(cm.pretty_format(sort_by_count = True, show_percents = True, truncate = 9)) # take off the triple string to print precision, recall, and F1 for each fold print('\tPrecision\tRecall\t\tF1') # print measures for each label for i, lab in enumerate(labels): print(lab, '\t', "{:10.3f}".format(precision_list[i]), \ "{:10.3f}".format(recall_list[i]), "{:10.3f}".format(F1_list[i])) # for each label add to the sums in the total list for i in range(num_labels): # for each label, add the 3 measures to the 3 lists of totals total_precision_list[i] += precision_list[i] total_recall_list[i] += recall_list[i] total_F1_list[i] += F1_list[i]
def confusion_matrix(test_data, classifier): correct_entailment_type = [] predicted_entailment_type = [] for test_case in test_data: correct_entailment_type.append(test_case[1]) result = classifier.classify(test_case[0]) predicted_entailment_type.append(result) cm = nltk.ConfusionMatrix(correct_entailment_type, predicted_entailment_type) return cm.pretty_format(sort_by_count=True, show_percents=True, truncate=9)
def test3(): def tag_list(tagged_sents): return [tag for sent in tagged_sents for (word, tag) in sent] def apply_tagger(tagger, corpus): return [tagger.tag(nltk.tag.untag(sent)) for sent in corpus] gold = tag_list(brown.tagged_sents(categories='editorial')) test = tag_list( apply_tagger(t2, brown.tagged_sents(categories='editorial'))) cm = nltk.ConfusionMatrix(gold, test) print(cm.pp(sort_by_count=True, show_precents=True, truncate=9))
def runTests(transitionDict, emissionDict, StagsArray, algorithm): if algorithm == BIGRAM_HMM: print("") print("------------>HMM bigram model for POS<-------------") if algorithm == ADD_SMOOTH: print("") print("------------>HMM Add-one smoothing for POS<-------------") if algorithm == PSEUDOWORDS: print("") print("------------>HMM Pseudo-Words for POS<-------------") if algorithm == PSEUDOWORDSANDSMOOTH: print("") print( "------------>HMM Pseudo-Words and SMOOTHING for POS<-------------" ) sumAllTestsErRatesKnown = 0 sumAllTestsErRatesUnkown = 0 sumAllTestsErRatesTotal = 0 count = 0 labelList = [] predictedList = [] for testSentence in testCorpus: count += 1 sentence = [word for word, tag in testSentence] tagList = viterbiAlg(sentence, emissionDict, transitionDict, StagsArray, algorithm) predictedList.extend(tagList) labelList.extend([tag for word, tag in testSentence]) erKnown, erUnKnown, totErate = ErrorRateForTest(testSentence, tagList) sumAllTestsErRatesKnown += erKnown sumAllTestsErRatesUnkown += erUnKnown sumAllTestsErRatesTotal += totErate sumAllTestsErRatesKnown = sumAllTestsErRatesKnown / count sumAllTestsErRatesUnkown = sumAllTestsErRatesUnkown / count sumAllTestsErRatesTotal = sumAllTestsErRatesTotal / count if (algorithm == PSEUDOWORDSANDSMOOTH): print("confusion Matrix for - ", algorithm) print("") confusionMatrix = nltk.ConfusionMatrix(labelList, predictedList) print( confusionMatrix.pretty_format(sort_by_count=True, show_percents=False)) print("-errorRateTest - Known : ", sumAllTestsErRatesKnown) print("-errorRateTest - UnKnown : ", sumAllTestsErRatesUnkown) print("-errorRateTest - total : ", sumAllTestsErRatesTotal) print("") print("--------------------------------------------")
def create_confusion_matrix(corpus, category, tagger): # get a list of the gold standard tags, and the tags set by the tagger. gold = tag_list(corpus.tagged_sents(categories=category)) test = tag_list( apply_tagger(tagger, corpus.tagged_sents(categories=category))) # create the confusion matrix and return it in a pretty-printed format. cm = nltk.ConfusionMatrix(gold, test) return cm.pretty_format(sort_by_count=True, show_percents=True, truncate=20)
def run(self): args = self.get_arguments() model_file = args[0] filenames = args[1:] if not os.path.exists(model_file): print("ERROR: model file '%s' doesn't exist" % model_file) return True model = SentimentModel.load(model_file) evaluator = ModelEvaluator(model, filenames) gold, test = evaluator.get_gold_comparison() print("Confusion matrix:") cm = nltk.ConfusionMatrix(gold, test) print(cm.pp(sort_by_count=True, show_percents=True)) summary = evaluator.evaluate() precision_sum, recall_sum, fmeasure_sum = 0, 0, 0 print(" TP TN FP FN precision recall f-measure") rowstr = "%(label)s %(tp)6s %(tn)6s %(fp)6s %(fn)6s %(precision)2.5f %(recall)2.5f %(fmeasure)2.5f" for sentiment, label in TEXT_LABELS.items(): print(rowstr % { "label": '%s:' % label.rjust(10), "tp": summary[sentiment]["true-positives"], "fp": summary[sentiment]["false-positives"], "tn": summary[sentiment]["true-negatives"], "fn": summary[sentiment]["false-negatives"], "precision": summary[sentiment]["precision"], "recall": summary[sentiment]["recall"], "fmeasure": summary[sentiment]["f-measure"], }) precision_sum += summary[sentiment]["precision"] recall_sum += summary[sentiment]["recall"] fmeasure_sum += summary[sentiment]["f-measure"] precision_avg = precision_sum / len(TEXT_LABELS) recall_avg = recall_sum / len(TEXT_LABELS) fmeasure_avg = fmeasure_sum / len(TEXT_LABELS) print(rowstr % { "label": "average:".rjust(11), "tp": "", "fp": "", "tn": "", "fn": "", "precision": precision_avg, "recall": recall_avg, "fmeasure": fmeasure_avg, }) return False
def classifierConfusionMatrix(self, posTokenizedPostsTest, negTokenizedPostsTest, posLIWCPosts, negLIWCPosts): gold = self.getFeatureSetForAllPosts(posTokenizedPostsTest, negTokenizedPostsTest, posLIWCPosts, negLIWCPosts) feature_set = self.removeClassesFromFeatures(gold) predictions = self.classifier.classify_many(feature_set) gs = [] for i in range(0, len(gold)): g, v = gold[i] gs += [v] return nltk.ConfusionMatrix(gs, predictions)
def calculateStats(self, i): '''Function to calculate all stats''' #calculate cm for this iteration ref = [] tagged = [] for f, e in self.testB: ref.append(self.classifierB.classify(f)) tagged.append(e) self.cm.append(nltk.ConfusionMatrix(ref, tagged)) #self.informativeFeatures.append(self.classifierB.most_informative_features(10)) print() #calculate precision and recall for this iteration for each category refsets = defaultdict(set) testsets = defaultdict(set) #allCount = 0 #noEventCount = 0 for n, (feats, label) in enumerate(self.testB): #allCount += 1 #if label == "geen_event": # noEventCount += 1 refsets[label].add(n) observed = self.classifierB.classify(feats) # uncomment voor wel_event #if label != "geen_event": # refsets["wel_event"].add(n) #if observed != "geen_event": # testsets["wel_event"].add(n) testsets[observed].add(n) #print("Accuracy geen_event (baseline) is", noEventCount/allCount) # self.accuracy.append( nltk.classify.accuracy(self.classifierB, self.testB)) #for elke category precision and recall berekenen. for category in self.categories: if category in testsets: self.result[category]["p"].append( nltk.metrics.precision(refsets[category], testsets[category])) self.result[category]["r"].append( nltk.metrics.recall(refsets[category], testsets[category])) self.result[category]["f"].append( nltk.metrics.f_measure(refsets[category], testsets[category])) else: self.result[category]["p"].append(float(0)) self.result[category]["r"].append(float(0)) self.result[category]["f"].append(float(0))