def evaluateFromFile(evalFile, sourceData, predictDir):
evalData = None
evalData = pomio.readEvaluationListFromCsv(evalFile)
assert evalData != None, "Exception reading evaluation data from " + str(
evalFile)
print "\nINFO: Eval file list = " + str(evalFile)
print "INFO: Source data = " + str(sourceData)
print "\nINFO 1st element in eval result::", evalData[0]
if predictDir.endswith("/") == False:
predictDir = predictDir + "/"
# Tells us what the elements of the results are
headers = [
"numCorrectPixels", "numberValidGroundTruthPixels",
"numberVoidGroundTruthPixels", "numberPixelsInImage"
]
results = None
confMat = None
# for each eval pair (prediction labels and ground truth labels) do pixel count
for idx in range(0, len(evalData)):
predictFile = evalData[idx][0]
gtFile = evalData[idx][1]
gt = loadReferenceGroundTruthLabels(sourceData, gtFile)
predict = loadPredictionImageLabels(predictDir + predictFile)
result = evaluatePrediction(predict, gt, gtFile)
cmat = evaluateConfusionMatrix(predict, gt)
if idx == 0:
results = result
confMat = cmat
else:
results += result
confMat += cmat
# Aggregate results
perClass = evaluateClassPerformance(confMat)
print "Processed total of ", len(evalData), "predictions:"
print " Average accuracy per pixel: ", 100.0 * results[0] / float(
results[1])
print " Average accuracy per class: ", perClass.mean()
print " Accuracy per class: "
print pandas.DataFrame( perClass.reshape((1,len(perClass))), \
columns=pomio.getClasses()[:-1] ).to_string()
print ""
print " Confusion matrix (row=gt, col=predicted): "
print pandas.DataFrame( confMat, columns=pomio.getClasses()[:-1], \
index=pomio.getClasses()[:-1] ).to_string()
print "Processing complete."
def reportAccuracy( exptName, labs, predlabs ):
print exptName, ' accuracy (frac correct) = ', np.mean(predlabs==labs)
apc = accuracyPerClass( labs, predlabs )
print ' - average accuracy per class = ', apc.mean()
for i in range(pomio.getNumClasses()):
print ' %s: %f' %( pomio.getClasses()[i], apc[i] )
clp, clnum = classProportions( labs )
print ' - class proportions in %s:' % exptName
for i in range(pomio.getNumClasses()):
print ' %15s: %.6f (%6d examples)' %( pomio.getClasses()[i], clp[i], clnum[i] )
def reportAccuracy(exptName, labs, predlabs):
print exptName, ' accuracy (frac correct) = ', np.mean(predlabs == labs)
apc = accuracyPerClass(labs, predlabs)
print ' - average accuracy per class = ', apc.mean()
for i in range(pomio.getNumClasses()):
print ' %s: %f' % (pomio.getClasses()[i], apc[i])
clp, clnum = classProportions(labs)
print ' - class proportions in %s:' % exptName
for i in range(pomio.getNumClasses()):
print ' %15s: %.6f (%6d examples)' % (pomio.getClasses()[i],
clp[i], clnum[i])
if args.labs == None:
labs = None
else:
if args.labs.endswith('.pkl'):
labs = pomio.unpickleObject( args.labs )
else:
labs = pomio.readMatFromCSV( args.labs ).astype(np.int32)
# show labels
if labs != None:
plt.figure()
plt.hist( labs, pomio.getNumClasses() )
plt.title('Class counts')
plt.xticks( range(pomio.getNumClasses()),
pomio.getClasses()[:pomio.getNumClasses()],
size='small' )
# show at most 9 features at once
fstarts = range( args.nstart, D, args.nshow )
plt.figure()
for fs in fstarts:
fend = min(D-1,fs+args.nshow-1)
print ' Displaying features %d-%d:' % (fs, fend)
rng = range( fs, fend+1 )
fnames = [ 'F%d' % x for x in rng ]
amntools.gplotmatrix( ftrs[:,rng], labs, featureNames=fnames )
plt.waitforbuttonpress()
if args.labs == None:
labs = None
else:
if args.labs.endswith('.pkl'):
labs = pomio.unpickleObject(args.labs)
else:
labs = pomio.readMatFromCSV(args.labs).astype(np.int32)
# show labels
if labs != None:
plt.figure()
plt.hist(labs, pomio.getNumClasses())
plt.title('Class counts')
plt.xticks(range(pomio.getNumClasses()),
pomio.getClasses()[:pomio.getNumClasses()],
size='small')
# show at most 9 features at once
fstarts = range(args.nstart, D, args.nshow)
plt.figure()
for fs in fstarts:
fend = min(D - 1, fs + args.nshow - 1)
print ' Displaying features %d-%d:' % (fs, fend)
rng = range(fs, fend + 1)
fnames = ['F%d' % x for x in rng]
amntools.gplotmatrix(ftrs[:, rng], labs, featureNames=fnames)
plt.waitforbuttonpress()
plt.interactive(0)
