def classify_images(net, transformer, img_test_names, img_test_labels, settings):
pb = ProgressBar(len(img_test_names))
cm = ConfusionMatrix(li.labels)
pred_hit = np.zeros(len(img_test_names))
class_preds = []
for i, (id_, true_class) in enumerate(zip(img_test_names, img_test_labels)):
pb.print_progress()
if (settings["bulk"]):
settings["test_dir"] = settings["bulk_dir"]
pred_class = classify_from_bulk(net, transformer, id_, true_class, settings)
else:
pred_class = classify_image(net, transformer, id_, settings)[0]
class_preds.append(pred_class)
if (pred_class == true_class):
pred_hit[i] = 1
cm.actualize(true_class, pred_class)
submission_file = print_submission(img_test_names, class_preds, settings)
accuracy = compute_accuracy(pred_hit)
return submission_file, cm.get_confusion_matrix(), accuracy
def __update__(self):
'''Updates self.CONFS to new ACTUAL or PREDS values'''
for model in self.PREDS:
assert len(self.ACTUAL)==len(self.PREDS[model]),"Predicted and actual data must be of equal length"
if len(set(self.PREDS[model]))>2:
self.CONFS.update( {model: ConfusionMatrix.ConfusionMatrix(actual=self.ACTUAL,pred=self.PREDS[model],bins=1000).getConfs()} )
else:
self.CONFS.update( {model: ConfusionMatrix.ConfusionMatrix(actual=self.ACTUAL,pred=self.PREDS[model])} )
def evaluate(self, test_data):
#forward_res = [self.feedforward(x) for (x,y) in test_data]
#print('forward_res \t',forward_res)
test_results = [(np.amax(self.feedforward(x)), (y))
for (x, y) in test_data]
test_results1 = [((1 if x > 0.5 else 0), y) for (x, y) in test_results]
#print('test_result:\t',test_results1)
total = sum(int(x == y) for (x, y) in test_results1)
tp, fp, tn, fn = cm.ComputeConfusionMatrix(test_results1)
print('True Positive:', tp)
print('False Positive:', fp)
print('True Negative:', tn)
print('False Negative:', fn)
return total
# dataset.print_label()
optimalParameter = True #determine if the optimal value for gamma and cost are used (True)
#or if personnalized values are used (False)
#-------PARAMETERS TO MODIFY (works only if optimalParameter = False-------
cost = 10 #optimal value: 10
gamma = 0.5 #optimal value: 0.5
#--- MAIN
print("Problem D")
if SVM:
print("Number of input data: " + str(NBR_POINT))
X_train, X_test, y_train, y_test = train_test_split(dataset.input,
dataset.label,
train_size=0.5,
random_state=2)
if optimalParameter:
svc = svm.run_svm_nonlinear(X_train, y_train, kernel="rbf")
else:
svc = svm.run_svm_nonlinear(X_train,
y_train,
kernel="rbf",
cost=cost,
gamma=gamma)
svm.plot_svc(svc, X_test, y_test)
cm = ConfusionMatrix.ConfusionMatrix(svc, X_test, y_test)
print("Support Vector Machine: ")
cm.print_matrix()
import matplotlib as mpl
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
NBR_EPOCH_MAX = 2000
PERCEPTRON = True
SVM = True
dataset = data.data1
#--- MAIN
print("Problem A")
if PERCEPTRON:
print("Perceptron learning rule: ")
(W, b) = per.train_nn_perceptron(dataset.inputON, dataset.inputOFF, NBR_EPOCH_MAX, "hardlims", 1)
W = np.append(W, b)
print("W = ")
print(W)
per.plot_perceptron(dataset, W)
if SVM:
# tuned_parameters = [{'C': [0.001, 0.01, 0.1, 1, 5, 10, 100]}]
# clf = GridSearchCV(SVC(kernel='linear'), tuned_parameters, cv=5, scoring='accuracy')
# clf.fit(dataset.input, dataset.label)
# print("Best param: ")
# print(clf.best_params_)
svc = svm.run_svm(dataset, cost=1, kernel="linear") #cost value doesn't have any impact here
svm.plot_svc(svc, dataset.input,dataset.label)
cm = ConfusionMatrix.ConfusionMatrix(svc, dataset)
cm.print_matrix()
def evaluate_filelist(matchedfilenames, excludedhtidlist):
global consensus, groundtruthdir, filewordcounts
smoothederrors = dict()
unsmoothederrors = dict()
smoothedcorrect = dict()
unsmoothedcorrect = dict()
coalescederrors = dict()
coalescedcorrect = dict()
totalgt = dict()
roughaccurate = 0
roughnotaccurate = 0
smoothaccurate = 0
smoothnotaccurate = 0
coalescedaccurate = 0
coalescednotaccurate = 0
# The correct dictionaries pair a genre code (in the original) to a number of times it was correctly
# identified
# The error dictionaries map a tuple of (correct code, error code) to a number of times it occurred.
truesequences = dict()
predictedsequences = dict()
accuracies = dict()
metadatatable = dict()
symptoms = ["weakconfirmation", "weakdenial", "strongconfirmation", "strongdenial", "modelagrees", "modeldisagrees"]
for symptom in symptoms:
metadatatable[symptom] = dict()
metadatatable["numberofchunks"] = dict()
# metadatatable["fictonon"] = dict()
# metadatatable["bio"] = dict()
for pfile, gtfile in matchedfilenames.items():
htid = gtfile[0:-4]
if htid in excludedhtidlist:
continue
# The predictionfile has three columns, of which the second
# is an unsmoothed prediction and the third is smoothed
smoothlist = consensus[pfile]
# roughlist = list()
# detailedprobabilities = list()
# pfilepath = os.path.join(predictdir, pfile)
# with open(pfilepath,encoding = "utf-8") as f:
# filelines = f.readlines()
# for line in filelines:
# line = line.rstrip()
# fields = line.split('\t')
# roughlist.append(fields[1])
# smoothlist.append(fields[2])
# if len(fields) > 5:
# detailedprobabilities.append("\t".join(fields[5:]))
# # The prediction file has this format:
# # pagenumber roughgenre smoothgenre many ... detailed predictions
# # fields 3 and 4 will be predictions for dummy genres "begin" and "end"
correctlist = list()
gtfilepath = os.path.join(groundtruthdir, gtfile)
with open(gtfilepath,encoding = "utf-8") as f:
filelines = f.readlines()
for line in filelines:
line = line.rstrip()
fields = line.split('\t')
correctlist.append(fields[1])
assert len(correctlist) == len(smoothlist)
if countwords:
tuplelist = filewordcounts[htid]
wordsperpage = [x[1] for x in tuplelist]
else:
wordsperpage = list()
# Experiment.
oldgenre = ""
transitioncount = 0
biocount = 0
for agenre in smoothlist:
if agenre == "bio":
biocount += 1
if oldgenre == "fic" and (agenre == "non" or agenre =="bio"):
transitioncount += 1
oldgenre = agenre
# fictionfilepath = os.path.join(thefictiondir, pfile)
# poetryfilepath = os.path.join(thepoedir, pfile)
# mainmodel = cascades.read_probabilities(detailedprobabilities)
mostlydrapoe, probablybiography, probablyfiction, notdrama, notfiction = cascades.choose_cascade(htid, smoothlist)
# # This function returns three boolean values which will help us choose a specialized model
# # to correct current predictions. This scheme is called "cascading classification," thus
# # we are "choosing a cascade."
# Make defensive copy
adjustedlist = [x for x in smoothlist]
if notdrama:
adjustedlist = nix_a_genre(adjustedlist, "dra", secondthoughts[pfile])
if notfiction:
adjustedlist = nix_a_genre(adjustedlist, "fic", secondthoughts[pfile])
# if thepoedir != "n" and thefictiondir != "n":
# numberoftrues = sum([mostlydrapoe, probablybiography, probablyfiction])
# if numberoftrues == 1:
# if mostlydrapoe and thepoedir != "n":
# adjustedlist, mainmodel = cascades.drapoe_cascade(adjustedlist, mainmodel, poetryfilepath)
# elif probablybiography:
# adjustedlist = cascades.biography_cascade(adjustedlist)
# elif probablyfiction and thefictiondir != "n":
# adjustedlist, mainmodel = cascades.fiction_cascade(adjustedlist, mainmodel, fictionfilepath)
if tocoalesce:
coalescedlist, numberofdistinctsequences = Coalescer.coalesce(adjustedlist)
# This function simplifies our prediction by looking for cases where a small
# number of pages in genre X are surrounded by larger numbers of pages in
# genre Y. This is often an error, and in cases where it's not technically
# an error it's a scale of variation we usually want to ignore. However,
# we will also record detailed probabilities for users who *don't* want to
# ignore these
else:
coalescedlist = adjustedlist
dummy, numberofdistinctsequences = Coalescer.coalesce(adjustedlist)
metadataconfirmation = cascades.metadata_check(htid, coalescedlist)
# Now that we have adjusted
for key, value in metadataconfirmation.items():
metadatatable[key][htid] = value
metadatatable["numberofchunks"][htid] = log(numberofdistinctsequences + 1)
# metadatatable["fictonon"][htid] = transitioncount
# metadatatable["bio"][htid] = biocount / len(roughlist)
# This is significant. We don't want to overpenalize long books, but there is
# a correlation between the number of predicted genre shifts and inaccuracy.
# So we take the log.
totaltruegenre, correctbygenre, errorsbygenre, accurate, inaccurate = compare_two_lists(correctlist, smoothlist, wordsperpage, countwords)
add_dictionary(smoothederrors, errorsbygenre)
add_dictionary(smoothedcorrect, correctbygenre)
add_dictionary(totalgt, totaltruegenre)
# Only do this for one comparison
smoothaccurate += accurate
smoothnotaccurate += inaccurate
if ("index", "non") in errorsbygenre:
if errorsbygenre[("index", "non")] > 2:
print("Index fail: " + htid + " " + str(errorsbygenre[("index", "non")]))
# totaltruegenre, correctbygenre, errorsbygenre, accurate, inaccurate = compare_two_lists(correctlist, roughlist, wordsperpage, countwords)
# add_dictionary(unsmoothederrors, errorsbygenre)
# add_dictionary(unsmoothedcorrect, correctbygenre)
# roughaccurate += accurate
# roughnotaccurate += inaccurate
totaltruegenre, correctbygenre, errorsbygenre, accurate, inaccurate = compare_two_lists(correctlist, coalescedlist, wordsperpage, countwords)
add_dictionary(coalescederrors, errorsbygenre)
add_dictionary(coalescedcorrect, correctbygenre)
coalescedaccurate += accurate
coalescednotaccurate += inaccurate
truesequences[gtfile] = correctlist
predictedsequences[gtfile] = coalescedlist
thisaccuracy = accurate / (accurate + inaccurate)
accuracies[htid] = thisaccuracy
# Now we need to interpret the dictionaries.
for genre, count in totalgt.items():
print()
print(genre.upper() + " : " + str(count))
if count < 1:
continue
print()
print("SMOOTHED PREDICTION, " + str(count) + " | " + genre)
print("Correctly identified: " + str(smoothedcorrect.get(genre, 0) / count))
print("Errors: ")
for key, errorcount in smoothederrors.items():
gt, predict = key
if gt == genre:
print(predict + ": " + str(errorcount) + " " + str (errorcount/count))
print()
print("COALESCED PREDICTION, " + str(count) + " | " + genre)
print("Correctly identified: " + str(coalescedcorrect.get(genre, 0) / count))
print("Errors: ")
for key, errorcount in coalescederrors.items():
gt, smoothed = key
if gt == genre:
print(smoothed + ": " + str(errorcount) + " " + str (errorcount/count))
# roughaccuracy = roughaccurate / (roughaccurate + roughnotaccurate)
smoothaccuracy = smoothaccurate / (smoothaccurate + smoothnotaccurate)
coalaccuracy = coalescedaccurate / (coalescedaccurate + coalescednotaccurate)
confusion = ConfusionMatrix.confusion_matrix(coalescedcorrect, coalescederrors)
return metadatatable, accuracies, smoothaccuracy, coalaccuracy
def Test_Dataset(TaskID, datasetBase, datasetConfig, ModelFileName, testData, true_labels, modelInLogDir = False):
#ROOT_DIR = os.getcwd()
ROOT_DIR = datasetBase + "/LOGS"
LOG_DIR = ROOT_DIR + "/logs"
if(modelInLogDir):
MODEL_DIR = LOG_DIR
else:
MODEL_DIR = ROOT_DIR + "/Models"
LOG_DIR_FOLDER = LOG_DIR + "/Test_" + TaskID
print("Test_" + TaskID)
HistDumpFile = "History_" + TaskID + ".txt"
ConfusionMatrixImg = "Confusion_" + TaskID + ".png"
ResultFile = "Result_" + TaskID + ".csv"
ReportFile = "Report" + TaskID + ".csv"
model = load_model(MODEL_DIR + "/" + ModelFileName)
testData = np.asarray(testData)
if(len(testData.shape) == 3):
testData = np.expand_dims(testData, -1)
total_result = []
total_result_label = []
total_true_label = []
num_classes = datasetConfig.NUM_CLASS
classes = datasetConfig.CLASSES
if(not os.path.isdir(LOG_DIR_FOLDER)):
os.mkdir(LOG_DIR_FOLDER)
preds = model.predict(testData)
pred_result = preds.argmax(axis = 1)
result = np.zeros(pred_result.shape)
#print(true_Label)
#print(pred_result)
for i in range(len(result)):
result[i] = true_labels[i][pred_result[i]]
true_labels = np.asarray(true_labels)
true_labels_binary = true_labels.argmax(axis = 1)
print(result)
accuracy = result.sum() / len(result)
print(accuracy)
# accuracy = get_Accucary(total_result.flatten())
con_matrix = confusion_matrix(true_labels_binary, pred_result)
ConfusionMatrix.plot_confusion_matrix(con_matrix, classes, saveImgFile = True, ImgFilename =LOG_DIR_FOLDER + "/" +ConfusionMatrixImg)
report = classification_report(true_labels_binary, pred_result, target_names=classes)
save_TestHistoryLog(TaskID, ROOT_DIR, ModelFileName, accuracy, datasetConfig)
save_TestResultLog(TaskID, ROOT_DIR, result, ModelFileName,datasetConfig.DATASET)
f = open(LOG_DIR_FOLDER + "/" + ReportFile, 'w')
f.write(report)
#f.write(np.array2string(total_result))
f.close()
del model
gc.collect()
K.clear_session()
print("Test Done ! ")
return accuracy
def main():
seed(1)
spam_dataset = extract_full_dataset()
shuffle(spam_dataset.values)
# print(spam_dataset.shape)
trainingSet, testingSet = train_test_split(spam_dataset, test_size=0.2)
dataset_k_split = kfold_split(5)
spam_accuracy = []
train_accuracy = []
#trainingSet, testingSet = get_training_testing_split(spam_dataset, dataset_k_split, i)
trainingSet = trainingSet.values
testingSet = testingSet.values
# print(trainingSet.shape, testingSet.shape)
y_test = testingSet[:, -1]
y_test = y_test[:, None]
x_test = testingSet[:, 0:len(testingSet[0]) - 1]
y_train = trainingSet[:, -1]
y_train = y_train[:, None]
x_train = trainingSet[:, 0:len(trainingSet[0]) - 1]
# print(x_train.shape, x_test.shape, y_test.shape, x_test.shape)
scaler = preprocessing.StandardScaler()
scaler.fit(x_train)
train = scaler.transform(x_train)
scaler = preprocessing.StandardScaler()
scaler.fit(x_test)
test = scaler.transform(x_test)
# Training error
best_w = gradient_descent(train, y_train)
# training error section
x_b_training = np.c_[np.ones(train.shape[0]), train]
train_y_predict = x_b_training.dot(best_w)
train_y_sigmoid = sigmoid(train_y_predict)
train_accu, train_normalized_prediction = evaluate_prediction_accuracy(
train_y_sigmoid, y_train, 0.4)
train_accuracy.append(train_accu)
# testing error section
# testing error section
X_new_testing = np.c_[np.ones(test.shape[0]), test]
test_y_predict = X_new_testing.dot(best_w)
test_y_sigmoid = sigmoid(test_y_predict)
# print(test_y_predict)
test_accu, test_normalized_prediction = evaluate_prediction_accuracy(
test_y_sigmoid, y_test, 0.4)
spam_accuracy.append(test_accu)
#print(y_test.shape, test_y_sigmoid.shape)
y_test_list = []
test_y_sigmoid_list = []
for val in y_test:
y_test_list.append(val)
for val in test_y_sigmoid:
test_y_sigmoid_list.append(val)
ConfusionMatrix.confusion_matrix(y_test_list, test_normalized_prediction,
test_y_sigmoid, True)
print("Training error is", np.mean(train_accuracy))
print("Testing accuracy is", np.mean(spam_accuracy))
import pandas as pd
import KnnClassifier as knn
import ConfusionMatrix as Cm
import RandomSubsampling as Rs
data = pd.read_csv("Iris.csv")
data = data.drop('Id', axis=1)
classifier = knn.KnnClassifier()
cm = Cm.ConfusionMatrix(len(data['Species'].unique()))
Random_Sub = Rs.Random_Subsampling()
X = data.drop('Species', axis=1)
y = data['Species']
error, accuracy = Random_Sub.Random_Subsampling(classifier, cm, X, y)
cm.show_conf_matrix()
print()
print("Accuracy: ", accuracy)
print("Error: ", error)
NBR_EPOCH_MAX = 2000
PERCEPTRON = True
SVM = True
dataset = data.data1
optimalParameter = True #determine if the optimal value for cost is used (True)
#or if personnalized value is used (False)
#-------PARAMETERS TO MODIFY (works only if optimalParameter = False-------
cost = 1 #optimal value: 100
#--- MAIN
print("Problem A")
if PERCEPTRON:
print("Perceptron learning rule: ")
(W, b) = per.train_nn_perceptron(dataset.inputON, dataset.inputOFF, NBR_EPOCH_MAX, "hardlims", 1)
W = np.append(W, b)
print("W = ")
print(W)
per.plot_perceptron(dataset, W)
if SVM:
if (optimalParameter):
svc = svm.run_svm_linear(dataset.input, dataset.label , kernel="linear")
else:
svc = svm.run_svm_linear(dataset.input, dataset.label , kernel="linear", cost=1)
svm.plot_svc(svc,dataset.input,dataset.label)
cm = ConfusionMatrix.ConfusionMatrix(svc, dataset.input,dataset.label)
print("Support Vector Machine: ")
cm.print_matrix()
for forest_idx in range(NUM_CLASSES):
choices = decision_forest_vote_weighted(forest[forest_idx],
weights[:, forest_idx], x)
vote_block[:, forest_idx] = choices
prediction = np.zeros((x.shape[0], 1))
for vote_idx in range(vote_block.shape[0]):
prediction[vote_idx] = int(np.argmax(vote_block[vote_idx])) + 1
else:
print "Testing Tree by Max Depth"
prediction = TestTreesByDepth(trees, x, ambiguityHandlingStyle)
print "Test Done"
print ""
cm = ConfusionMatrix(prediction, y, NUM_CLASSES)
error = sampleError(prediction, y)
measures = MeasurePerFold(cm, NUM_CLASSES)
print "Confusion Matrix"
PrintMatrix(cm)
print ""
PrintMatrix(measures)
print "Recall = %.3f, Precision = %.3f, F1 = %.3f, CR = %.3f" % (
np.sum(measures[0, :] / 6.0), np.sum(measures[1, :] / 6.0),
np.sum(measures[2, :] / 6.0), np.sum(measures[3, :] / 6.0))
print ""
print "Error Rate: %.3f" % error
print ""
for reading in listoftuples:
readera = reading[0]
predictedgenres = reading[1]
divergence = comparelists(predictedgenres, truegenres, genremistakes, correctbygenre, wordcounts)
totaldivergence += divergence
agreement = (potentialcomparisons - totaldivergence)
agreementpercent = agreement / potentialcomparisons
volumepercents[htid] = agreementpercent
overallcomparisons += potentialcomparisons
overallagreement += agreement
print("Average human agreement: " + str(overallagreement / overallcomparisons))
with open("/Users/tunder/Dropbox/pagedata/interrater/HumanDissensus.tsv", mode="w", encoding = "utf-8") as f:
f.write("htid\tagreement\n")
for key, value in volumepercents.items():
outline = utils.pairtreelabel(key) + "\t" + str(value) + "\n"
f.write(outline)
import ConfusionMatrix
ConfusionMatrix.confusion_matrix(correctbygenre, genremistakes)
potentialcomparisons = nummaps * sum(wordcounts)
totaldivergence = 0
for reading in listoftuples:
readera = reading[0]
predictedgenres = reading[1]
divergence = comparelists(predictedgenres, truegenres, genremistakes,
correctbygenre, wordcounts)
totaldivergence += divergence
agreement = (potentialcomparisons - totaldivergence)
agreementpercent = agreement / potentialcomparisons
volumepercents[htid] = agreementpercent
overallcomparisons += potentialcomparisons
overallagreement += agreement
print("Average human agreement: " + str(overallagreement / overallcomparisons))
with open("/Users/tunder/Dropbox/pagedata/interrater/HumanDissensus.tsv",
mode="w",
encoding="utf-8") as f:
f.write("htid\tagreement\n")
for key, value in volumepercents.items():
outline = utils.pairtreelabel(key) + "\t" + str(value) + "\n"
f.write(outline)
import ConfusionMatrix
ConfusionMatrix.confusion_matrix(correctbygenre, genremistakes)
y_test_predictions = np.zeros((x_test.shape[0], 1))
for vote_idx in range(vote_block.shape[0]):
y_test_predictions[vote_idx] = int(
np.argmax(vote_block[vote_idx])) + 1
depth_correct = 0
for prediction_idx in range(0, y_test_predictions.shape[0]):
if y_test_predictions[prediction_idx] == y_test[
prediction_idx]:
depth_correct += 1
depth_correct /= float(len(y_test_predictions))
stacked_sample_error_depth += 1 - depth_correct
stacked_confusion_matrices_depth += ConfusionMatrix(
y_test_predictions, y_test, NUM_CLASSES)
confusion_matrices_depth.append(
ConfusionMatrix(y_test_predictions, y_test, NUM_CLASSES))
print 'Fold accuracy:', depth_correct
else:
forestList = []
y_predictions = np.zeros((y_test.shape[0], 1), dtype=np.int32)
for class_idx in range(NUM_CLASSES):
class_num = class_idx + 1
forestList.append([])
for tree_idx in range(NUM_TREES_IN_FOREST):
x_sample, y_sample = sample(x_train, y_train,
SAMPLE_PROPORTION)
train_targets = PreProcess(y_sample, class_num)
#print "validation_set", len(validation_set), "and ","training_set", len(training_set)
#######################################################################
### Training the Model
### We are using Naive Bayes Algorithms to Train the Model
### Now that we have our training set, we can train our classifier
########################################################################
classifier = nltk.NaiveBayesClassifier.train(training_set)
## initialize the Confusion Matrix
myConfusionMatrix = ConfusionMatrix() #create a confusion matrix
#Create a Confusion Matrix
for tweet,label in (validation_set):
getLabel = classifier.classify((tweet))
myConfusionMatrix.matrix[int(label)][int(getLabel)] += 1
print "-----------------"
#Calculate precision,recall, fscore
calculate_precision_recall_fscore(myConfusionMatrix.matrix)
Accuracy = nltk.classify.accuracy(classifier, validation_set)
def evaluate_filelist(matchedfilenames, excludedhtidlist):
global consensus, groundtruthdir, filewordcounts
smoothederrors = dict()
unsmoothederrors = dict()
smoothedcorrect = dict()
unsmoothedcorrect = dict()
coalescederrors = dict()
coalescedcorrect = dict()
totalgt = dict()
roughaccurate = 0
roughnotaccurate = 0
smoothaccurate = 0
smoothnotaccurate = 0
coalescedaccurate = 0
coalescednotaccurate = 0
# The correct dictionaries pair a genre code (in the original) to a number of times it was correctly
# identified
# The error dictionaries map a tuple of (correct code, error code) to a number of times it occurred.
truesequences = dict()
predictedsequences = dict()
accuracies = dict()
metadatatable = dict()
symptoms = [
"weakconfirmation", "weakdenial", "strongconfirmation", "strongdenial",
"modelagrees", "modeldisagrees"
]
for symptom in symptoms:
metadatatable[symptom] = dict()
metadatatable["numberofchunks"] = dict()
# metadatatable["fictonon"] = dict()
# metadatatable["bio"] = dict()
for pfile, gtfile in matchedfilenames.items():
htid = gtfile[0:-4]
if htid in excludedhtidlist:
continue
# The predictionfile has three columns, of which the second
# is an unsmoothed prediction and the third is smoothed
smoothlist = consensus[pfile]
# roughlist = list()
# detailedprobabilities = list()
# pfilepath = os.path.join(predictdir, pfile)
# with open(pfilepath,encoding = "utf-8") as f:
# filelines = f.readlines()
# for line in filelines:
# line = line.rstrip()
# fields = line.split('\t')
# roughlist.append(fields[1])
# smoothlist.append(fields[2])
# if len(fields) > 5:
# detailedprobabilities.append("\t".join(fields[5:]))
# # The prediction file has this format:
# # pagenumber roughgenre smoothgenre many ... detailed predictions
# # fields 3 and 4 will be predictions for dummy genres "begin" and "end"
correctlist = list()
gtfilepath = os.path.join(groundtruthdir, gtfile)
with open(gtfilepath, encoding="utf-8") as f:
filelines = f.readlines()
for line in filelines:
line = line.rstrip()
fields = line.split('\t')
correctlist.append(fields[1])
assert len(correctlist) == len(smoothlist)
if countwords:
tuplelist = filewordcounts[htid]
wordsperpage = [x[1] for x in tuplelist]
else:
wordsperpage = list()
# Experiment.
oldgenre = ""
transitioncount = 0
biocount = 0
for agenre in smoothlist:
if agenre == "bio":
biocount += 1
if oldgenre == "fic" and (agenre == "non" or agenre == "bio"):
transitioncount += 1
oldgenre = agenre
# fictionfilepath = os.path.join(thefictiondir, pfile)
# poetryfilepath = os.path.join(thepoedir, pfile)
# mainmodel = cascades.read_probabilities(detailedprobabilities)
mostlydrapoe, probablybiography, probablyfiction, notdrama, notfiction = cascades.choose_cascade(
htid, smoothlist)
# # This function returns three boolean values which will help us choose a specialized model
# # to correct current predictions. This scheme is called "cascading classification," thus
# # we are "choosing a cascade."
# Make defensive copy
adjustedlist = [x for x in smoothlist]
if notdrama:
adjustedlist = nix_a_genre(adjustedlist, "dra",
secondthoughts[pfile])
if notfiction:
adjustedlist = nix_a_genre(adjustedlist, "fic",
secondthoughts[pfile])
# if thepoedir != "n" and thefictiondir != "n":
# numberoftrues = sum([mostlydrapoe, probablybiography, probablyfiction])
# if numberoftrues == 1:
# if mostlydrapoe and thepoedir != "n":
# adjustedlist, mainmodel = cascades.drapoe_cascade(adjustedlist, mainmodel, poetryfilepath)
# elif probablybiography:
# adjustedlist = cascades.biography_cascade(adjustedlist)
# elif probablyfiction and thefictiondir != "n":
# adjustedlist, mainmodel = cascades.fiction_cascade(adjustedlist, mainmodel, fictionfilepath)
if tocoalesce:
coalescedlist, numberofdistinctsequences = Coalescer.coalesce(
adjustedlist)
# This function simplifies our prediction by looking for cases where a small
# number of pages in genre X are surrounded by larger numbers of pages in
# genre Y. This is often an error, and in cases where it's not technically
# an error it's a scale of variation we usually want to ignore. However,
# we will also record detailed probabilities for users who *don't* want to
# ignore these
else:
coalescedlist = adjustedlist
dummy, numberofdistinctsequences = Coalescer.coalesce(adjustedlist)
metadataconfirmation = cascades.metadata_check(htid, coalescedlist)
# Now that we have adjusted
for key, value in metadataconfirmation.items():
metadatatable[key][htid] = value
metadatatable["numberofchunks"][htid] = log(numberofdistinctsequences +
1)
# metadatatable["fictonon"][htid] = transitioncount
# metadatatable["bio"][htid] = biocount / len(roughlist)
# This is significant. We don't want to overpenalize long books, but there is
# a correlation between the number of predicted genre shifts and inaccuracy.
# So we take the log.
totaltruegenre, correctbygenre, errorsbygenre, accurate, inaccurate = compare_two_lists(
correctlist, smoothlist, wordsperpage, countwords)
add_dictionary(smoothederrors, errorsbygenre)
add_dictionary(smoothedcorrect, correctbygenre)
add_dictionary(totalgt, totaltruegenre)
# Only do this for one comparison
smoothaccurate += accurate
smoothnotaccurate += inaccurate
if ("index", "non") in errorsbygenre:
if errorsbygenre[("index", "non")] > 2:
print("Index fail: " + htid + " " +
str(errorsbygenre[("index", "non")]))
# totaltruegenre, correctbygenre, errorsbygenre, accurate, inaccurate = compare_two_lists(correctlist, roughlist, wordsperpage, countwords)
# add_dictionary(unsmoothederrors, errorsbygenre)
# add_dictionary(unsmoothedcorrect, correctbygenre)
# roughaccurate += accurate
# roughnotaccurate += inaccurate
totaltruegenre, correctbygenre, errorsbygenre, accurate, inaccurate = compare_two_lists(
correctlist, coalescedlist, wordsperpage, countwords)
add_dictionary(coalescederrors, errorsbygenre)
add_dictionary(coalescedcorrect, correctbygenre)
coalescedaccurate += accurate
coalescednotaccurate += inaccurate
truesequences[gtfile] = correctlist
predictedsequences[gtfile] = coalescedlist
thisaccuracy = accurate / (accurate + inaccurate)
accuracies[htid] = thisaccuracy
# Now we need to interpret the dictionaries.
for genre, count in totalgt.items():
print()
print(genre.upper() + " : " + str(count))
if count < 1:
continue
print()
print("SMOOTHED PREDICTION, " + str(count) + " | " + genre)
print("Correctly identified: " +
str(smoothedcorrect.get(genre, 0) / count))
print("Errors: ")
for key, errorcount in smoothederrors.items():
gt, predict = key
if gt == genre:
print(predict + ": " + str(errorcount) + " " +
str(errorcount / count))
print()
print("COALESCED PREDICTION, " + str(count) + " | " + genre)
print("Correctly identified: " +
str(coalescedcorrect.get(genre, 0) / count))
print("Errors: ")
for key, errorcount in coalescederrors.items():
gt, smoothed = key
if gt == genre:
print(smoothed + ": " + str(errorcount) + " " +
str(errorcount / count))
# roughaccuracy = roughaccurate / (roughaccurate + roughnotaccurate)
smoothaccuracy = smoothaccurate / (smoothaccurate + smoothnotaccurate)
coalaccuracy = coalescedaccurate / (coalescedaccurate +
coalescednotaccurate)
confusion = ConfusionMatrix.confusion_matrix(coalescedcorrect,
coalescederrors)
return metadatatable, accuracies, smoothaccuracy, coalaccuracy
# print depth_cr
stacked_sample_error_depth += 1 - (depth_cr)
stacked_sample_error_depth_min += 1 - (depth_min_cr)
stacked_sample_error_occ += 1 - (occ_correct)
if depth_cr > best_depth_cr:
best_depth_cr = depth_cr
best_trees_depth = DeepCopyTreeList(treeList)
if depth_min_cr > best_depth_min_cr:
best_depth_min_cr = depth_min_cr
best_trees_depth_min = DeepCopyTreeList(treeList)
if occ_correct > best_occ_correct:
best_occ_correct = occ_correct
best_trees_occ = DeepCopyTreeList(treeList)
confusion_matrices_depth.append(
ConfusionMatrix(predictions_by_depth, y_test, NUM_CLASSES))
confusion_matrices_depth_min.append(
ConfusionMatrix(predictions_by_min_depth, y_test, NUM_CLASSES))
confusion_matrices_occ.append(
ConfusionMatrix(predictions_by_occ, y_test, NUM_CLASSES))
stacked_confusion_matrices_depth += ConfusionMatrix(
predictions_by_depth, y_test, NUM_CLASSES)
stacked_confusion_matrices_depth_min += ConfusionMatrix(
predictions_by_min_depth, y_test, NUM_CLASSES)
stacked_confusion_matrices_occ += ConfusionMatrix(predictions_by_occ,
y_test, NUM_CLASSES)
depth_parameters = [
np.zeros(NUM_CLASSES, dtype=np.float64),
np.zeros(NUM_CLASSES, dtype=np.float64),
