def getConfusionMatrixForImageList(classNum,
predfileList,
truefileList,
evalLabels=LABELS):
assert (len(predfileList) == len(truefileList))
enc = clr_trans
cm = np.zeros(shape=[classNum, classNum], dtype=np.uint64)
print('CSUPPORT:', CSUPPORT)
if CSUPPORT:
print('This is a fast C++ way evaluation.')
else:
print('This is a slow python way evaluation.')
for idx in tqdm(range(len(predfileList))):
predfile = predfileList[idx]
truefile = truefileList[idx]
imagePred = np.array(Image.open(predfile))
imageTrue = np.array(Image.open(truefile))
if len(imagePred.shape) == 3 and len(imageTrue.shape) == 3:
imagePred = enc.transform(imagePred, dtype=np.uint8)
imageTrue = enc.transform(imageTrue, dtype=np.uint8)
assert len(imagePred.shape) == 2 and len(imageTrue.shape) == 2
if CSUPPORT:
cm = addToConfusionMatrix.cEvaluatePair(imagePred, imageTrue, cm,
evalLabels)
else:
#slower python way
cm = calculateConfusionMatrix(cm, imagePred, imageTrue, evalLabels)
return cm
def evaluatePair(predictionImg, groundTruthImg, confMatrix, instanceStats,
perImageStats, args):
# Loading all resources for evaluation.
predictionNp = np.array(predictionImg)
groundTruthNp = np.array(groundTruthImg)
imgWidth = np.shape(predictionImg)[0]
imgHeight = np.shape(predictionImg)[1]
nbPixels = imgWidth * imgHeight
# Evaluate images
if (CSUPPORT):
# using cython
confMatrix = addToConfusionMatrix.cEvaluatePair(
predictionNp, groundTruthNp, confMatrix, args.evalLabels)
else:
# the slower python way
for (groundTruthImgPixel,
predictionImgPixel) in izip(np.ndarray.flatten(groundTruthImg),
np.ndarray.flatten(predictionImg)):
if (not groundTruthImgPixel in args.evalLabels):
printError(
"Unknown label with id {:}".format(groundTruthImgPixel))
confMatrix[groundTruthImgPixel][predictionImgPixel] += 1
return nbPixels
def evaluatePairPytorch(prediction, groundtruth, confMatrix, perImageStats,
args):
# Loading all resources for evaluation.
predictionImg = prediction
predictionNp = np.array(predictionImg)
groundTruthImg = groundtruth
groundTruthNp = np.array(groundTruthImg)
# Check for equal image sizes
if (predictionImg.size[0] != groundTruthImg.size[0]):
printError("Image widths are not equal.")
if (predictionImg.size[1] != groundTruthImg.size[1]):
printError("Image heights are not equal.")
if (len(predictionNp.shape) != 2):
printError("Predicted image has multiple channels.")
imgWidth = predictionImg.size[0]
imgHeight = predictionImg.size[1]
nbPixels = imgWidth * imgHeight
# Evaluate images
if (CSUPPORT):
# using cython
confMatrix = addToConfusionMatrix.cEvaluatePair(
predictionNp, groundTruthNp, confMatrix, args.evalLabels)
else:
# the slower python way
for (groundTruthImgPixel,
predictionImgPixel) in izip(groundTruthImg.getdata(),
predictionImg.getdata()):
if (not groundTruthImgPixel in args.evalLabels):
printError(
"Unknown label with id {:}".format(groundTruthImgPixel))
confMatrix[groundTruthImgPixel][predictionImgPixel] += 1
if args.evalPixelAccuracy:
notIgnoredLabels = [
l for l in args.evalLabels if not id2label[l].ignoreInEval
]
notIgnoredPixels = np.in1d(groundTruthNp,
notIgnoredLabels,
invert=True).reshape(groundTruthNp.shape)
erroneousPixels = np.logical_and(notIgnoredPixels,
(predictionNp != groundTruthNp))
perImageStats[predictionImgFileName] = {}
perImageStats[predictionImgFileName][
"nbNotIgnoredPixels"] = np.count_nonzero(notIgnoredPixels)
perImageStats[predictionImgFileName][
"nbCorrectPixels"] = np.count_nonzero(erroneousPixels)
return nbPixels
def evaluatePair(predictionImgFileName, groundTruthImgFileName, confMatrix,
instanceStats, perImageStats, args):
# Loading all resources for evaluation.
try:
predictionImg = Image.open(open(predictionImgFileName, 'rb'))
predictionNp = np.array(predictionImg)
except:
printError("Unable to load " + predictionImgFileName)
try:
groundTruthImg = Image.open(groundTruthImgFileName)
groundTruthNp = np.array(groundTruthImg)
except:
printError("Unable to load " + groundTruthImgFileName)
# load ground truth instances, if needed
if args.evalInstLevelScore:
groundTruthInstanceImgFileName = groundTruthImgFileName.replace(
"labelIds", "instanceIds")
try:
instanceImg = Image.open(groundTruthInstanceImgFileName)
instanceNp = np.array(instanceImg)
except:
printError("Unable to load " + groundTruthInstanceImgFileName)
#print("width",predictionImgFileName,predictionImg.size[0],groundTruthImg.size[0])
# Check for equal image sizes
if (predictionImg.size[0] != groundTruthImg.size[0]):
printError("Image widths of " + predictionImgFileName + " and " +
groundTruthImgFileName + " are not equal.")
if (predictionImg.size[1] != groundTruthImg.size[1]):
printError("Image heights of " + predictionImgFileName + " and " +
groundTruthImgFileName + " are not equal.")
if (len(predictionNp.shape) != 2):
printError("Predicted image has multiple channels.")
imgWidth = predictionImg.size[0]
imgHeight = predictionImg.size[1]
nbPixels = imgWidth * imgHeight
# Evaluate images
if (CSUPPORT):
# using cython
confMatrix = addToConfusionMatrix.cEvaluatePair(
predictionNp, groundTruthNp, confMatrix, args.evalLabels)
else:
# the slower python way
for (groundTruthImgPixel,
predictionImgPixel) in izip(groundTruthImg.getdata(),
predictionImg.getdata()):
if (not groundTruthImgPixel in args.evalLabels):
printError(
"Unknown label with id {:}".format(groundTruthImgPixel))
confMatrix[groundTruthImgPixel][predictionImgPixel] += 1
if args.evalInstLevelScore:
# Generate category masks
categoryMasks = {}
for category in instanceStats["categories"]:
categoryMasks[category] = np.in1d(
predictionNp,
instanceStats["categories"][category]["labelIds"]).reshape(
predictionNp.shape)
instList = np.unique(instanceNp[instanceNp > 1000])
for instId in instList:
labelId = int(instId / 1000)
label = id2label[labelId]
if label.ignoreInEval:
continue
mask = instanceNp == instId
instSize = np.count_nonzero(mask)
tp = np.count_nonzero(predictionNp[mask] == labelId)
fn = instSize - tp
weight = args.avgClassSize[label.name] / float(instSize)
tpWeighted = float(tp) * weight
fnWeighted = float(fn) * weight
instanceStats["classes"][label.name]["tp"] += tp
instanceStats["classes"][label.name]["fn"] += fn
instanceStats["classes"][label.name]["tpWeighted"] += tpWeighted
instanceStats["classes"][label.name]["fnWeighted"] += fnWeighted
category = label.category
if category in instanceStats["categories"]:
catTp = 0
catTp = np.count_nonzero(
np.logical_and(mask, categoryMasks[category]))
catFn = instSize - catTp
catTpWeighted = float(catTp) * weight
catFnWeighted = float(catFn) * weight
instanceStats["categories"][category]["tp"] += catTp
instanceStats["categories"][category]["fn"] += catFn
instanceStats["categories"][category][
"tpWeighted"] += catTpWeighted
instanceStats["categories"][category][
"fnWeighted"] += catFnWeighted
if args.evalPixelAccuracy:
notIgnoredLabels = [
l for l in args.evalLabels if not id2label[l].ignoreInEval
]
notIgnoredPixels = np.in1d(groundTruthNp,
notIgnoredLabels,
invert=True).reshape(groundTruthNp.shape)
erroneousPixels = np.logical_and(notIgnoredPixels,
(predictionNp != groundTruthNp))
perImageStats[predictionImgFileName] = {}
perImageStats[predictionImgFileName][
"nbNotIgnoredPixels"] = np.count_nonzero(notIgnoredPixels)
perImageStats[predictionImgFileName][
"nbCorrectPixels"] = np.count_nonzero(erroneousPixels)
return nbPixels
def evaluatePair(predictionImgFileName, groundTruthImgFileName, confMatrix, instanceStats, perImageStats, args):
# Loading all resources for evaluation.
try:
predictionImg = Image.open(predictionImgFileName)
predictionNp = np.array(predictionImg)
except:
printError("Unable to load " + predictionImgFileName)
try:
groundTruthImg = Image.open(groundTruthImgFileName)
groundTruthNp = np.array(groundTruthImg)
except:
printError("Unable to load " + groundTruthImgFileName)
# load ground truth instances, if needed
if args.evalInstLevelScore:
groundTruthInstanceImgFileName = groundTruthImgFileName.replace("labelIds","instanceIds")
try:
instanceImg = Image.open(groundTruthInstanceImgFileName)
instanceNp = np.array(instanceImg)
except:
printError("Unable to load " + groundTruthInstanceImgFileName)
# Check for equal image sizes
if (predictionImg.size[0] != groundTruthImg.size[0]):
printError("Image widths of " + predictionImgFileName + " and " + groundTruthImgFileName + " are not equal.")
if (predictionImg.size[1] != groundTruthImg.size[1]):
printError("Image heights of " + predictionImgFileName + " and " + groundTruthImgFileName + " are not equal.")
if ( len(predictionNp.shape) != 2 ):
printError("Predicted image has multiple channels.")
imgWidth = predictionImg.size[0]
imgHeight = predictionImg.size[1]
nbPixels = imgWidth*imgHeight
# Evaluate images
if (CSUPPORT):
# using cython
confMatrix = addToConfusionMatrix.cEvaluatePair(predictionNp, groundTruthNp, confMatrix, args.evalLabels)
else:
# the slower python way
for (groundTruthImgPixel,predictionImgPixel) in itertools.izip(groundTruthImg.getdata(),predictionImg.getdata()):
if (not groundTruthImgPixel in args.evalLabels):
printError("Unknown label with id {:}".format(groundTruthImgPixel))
confMatrix[groundTruthImgPixel][predictionImgPixel] += 1
if args.evalInstLevelScore:
# Generate category masks
categoryMasks = {}
for category in instanceStats["categories"]:
categoryMasks[category] = np.in1d( predictionNp , instanceStats["categories"][category]["labelIds"] ).reshape(predictionNp.shape)
instList = np.unique(instanceNp[instanceNp > 1000])
for instId in instList:
labelId = int(instId/1000)
label = id2label[ labelId ]
if label.ignoreInEval:
continue
mask = instanceNp==instId
instSize = np.count_nonzero( mask )
tp = np.count_nonzero( predictionNp[mask] == labelId )
fn = instSize - tp
weight = args.avgClassSize[label.name] / float(instSize)
tpWeighted = float(tp) * weight
fnWeighted = float(fn) * weight
instanceStats["classes"][label.name]["tp"] += tp
instanceStats["classes"][label.name]["fn"] += fn
instanceStats["classes"][label.name]["tpWeighted"] += tpWeighted
instanceStats["classes"][label.name]["fnWeighted"] += fnWeighted
category = label.category
if category in instanceStats["categories"]:
catTp = 0
catTp = np.count_nonzero( np.logical_and( mask , categoryMasks[category] ) )
catFn = instSize - catTp
catTpWeighted = float(catTp) * weight
catFnWeighted = float(catFn) * weight
instanceStats["categories"][category]["tp"] += catTp
instanceStats["categories"][category]["fn"] += catFn
instanceStats["categories"][category]["tpWeighted"] += catTpWeighted
instanceStats["categories"][category]["fnWeighted"] += catFnWeighted
if args.evalPixelAccuracy:
notIgnoredLabels = [l for l in args.evalLabels if not id2label[l].ignoreInEval]
notIgnoredPixels = np.in1d( groundTruthNp , notIgnoredLabels , invert=True ).reshape(groundTruthNp.shape)
erroneousPixels = np.logical_and( notIgnoredPixels , ( predictionNp != groundTruthNp ) )
perImageStats[predictionImgFileName] = {}
perImageStats[predictionImgFileName]["nbNotIgnoredPixels"] = np.count_nonzero(notIgnoredPixels)
perImageStats[predictionImgFileName]["nbCorrectPixels"] = np.count_nonzero(erroneousPixels)
return nbPixels
def evaluatePair(predictionImgFileName, groundTruthImgFileName, confMatrix,
instanceStats, perImageStats, args):
# Loading all resources for evaluation.
try:
predictionImg = Image.open(open(predictionImgFileName, 'rb'))
predictionNp = np.array(predictionImg)
except:
printError("Unable to load " + predictionImgFileName)
try:
groundTruthImg = Image.open(groundTruthImgFileName)
groundTruthNp = np.array(groundTruthImg)
except:
printError("Unable to load " + groundTruthImgFileName)
# load ground truth instances, if needed
if args.evalInstLevelScore:
groundTruthInstanceImgFileName = groundTruthImgFileName.replace(
"labelIds", "instanceIds")
try:
instanceImg = Image.open(groundTruthInstanceImgFileName)
instanceNp = np.array(instanceImg)
except:
printError("Unable to load " + groundTruthInstanceImgFileName)
# Check for equal image sizes
if (predictionImg.size[0] != groundTruthImg.size[0]):
printError("Image widths of " + predictionImgFileName + " and " +
groundTruthImgFileName + " are not equal.")
if (predictionImg.size[1] != groundTruthImg.size[1]):
printError("Image heights of " + predictionImgFileName + " and " +
groundTruthImgFileName + " are not equal.")
if (len(predictionNp.shape) != 2):
printError("Predicted image has multiple channels.")
imgWidth = predictionImg.size[0]
imgHeight = predictionImg.size[1]
nbPixels = imgWidth * imgHeight
imageSrc = groundTruthImgFileName.replace("_gtFine_labelIds.png",
"_leftImg8bit.png")
imgSrc = imageSrc.replace("gtFine", "leftImg8bit")
print(imgSrc)
srcImage = Image.open(open(imgSrc, 'rb'))
if (predictionImg.size[0] != groundTruthImg.size[0]
or predictionImg.size[1] != groundTruthImg.size[1]):
printError("Src sizes and pred sizes are different")
size = (groundTruthImg.size[0], groundTruthImg.size[1])
background = name2label['unlabeled'].color
newLabelImg = Image.new("RGBA", size, background)
# Evaluate images
if (CSUPPORT):
# using cython
confMatrix = addToConfusionMatrix.cEvaluatePair(
predictionNp, groundTruthNp, confMatrix, args.evalLabels)
else:
# the slower python way
# a drawer to draw into the image
#drawer = ImageDraw.Draw(newLabelImg)
#drawer.point()
for (groundTruthImgPixel,
predictionImgPixel) in izip(groundTruthImg.getdata(),
predictionImg.getdata()):
if (not groundTruthImgPixel in args.evalLabels):
printError(
"Unknown label with id {:}".format(groundTruthImgPixel))
confMatrix[groundTruthImgPixel][predictionImgPixel] += 1
drawer = ImageDraw.Draw(newLabelImg)
groundTruthPixels = groundTruthImg.load()
predictionImgPixels = predictionImg.load()
srcImagePixels = srcImage.load()
for i in range(groundTruthImg.size[0]):
for j in range(groundTruthImg.size[1]):
if (groundTruthPixels[i, j] != predictionImgPixels[i, j]):
groundTruthPixelij = groundTruthPixels[i, j]
#print(groundTruthPixelij)
labelId = groundTruthPixelij
if (groundTruthPixelij > 1000):
labelId = int(groundTruthPixelij / 1000)
#print(labelId)
label = id2label[labelId]
#print(label.color)
drawer.point([i, j], label.color)
else:
rgbaVal = srcImagePixels[i, j] + (170, )
drawer.point([i, j], rgbaVal)
dst = predictionImgFileName.replace("_labelIds.png", "_diffLabels.png")
print(dst)
newLabelImg.save(dst)
print("image save done", "", "", "")
if args.evalInstLevelScore:
# Generate category masks
categoryMasks = {}
for category in instanceStats["categories"]:
categoryMasks[category] = np.in1d(
predictionNp,
instanceStats["categories"][category]["labelIds"]).reshape(
predictionNp.shape)
instList = np.unique(instanceNp[instanceNp > 1000])
for instId in instList:
labelId = int(instId / 1000)
label = id2label[labelId]
if label.ignoreInEval:
continue
mask = instanceNp == instId
instSize = np.count_nonzero(mask)
tp = np.count_nonzero(predictionNp[mask] == labelId)
fn = instSize - tp
weight = args.avgClassSize[label.name] / float(instSize)
tpWeighted = float(tp) * weight
fnWeighted = float(fn) * weight
instanceStats["classes"][label.name]["tp"] += tp
instanceStats["classes"][label.name]["fn"] += fn
instanceStats["classes"][label.name]["tpWeighted"] += tpWeighted
instanceStats["classes"][label.name]["fnWeighted"] += fnWeighted
category = label.category
if category in instanceStats["categories"]:
catTp = 0
catTp = np.count_nonzero(
np.logical_and(mask, categoryMasks[category]))
catFn = instSize - catTp
catTpWeighted = float(catTp) * weight
catFnWeighted = float(catFn) * weight
instanceStats["categories"][category]["tp"] += catTp
instanceStats["categories"][category]["fn"] += catFn
instanceStats["categories"][category][
"tpWeighted"] += catTpWeighted
instanceStats["categories"][category][
"fnWeighted"] += catFnWeighted
if args.evalPixelAccuracy:
notIgnoredLabels = [
l for l in args.evalLabels if not id2label[l].ignoreInEval
]
notIgnoredPixels = np.in1d(groundTruthNp,
notIgnoredLabels,
invert=True).reshape(groundTruthNp.shape)
erroneousPixels = np.logical_and(notIgnoredPixels,
(predictionNp != groundTruthNp))
perImageStats[predictionImgFileName] = {}
perImageStats[predictionImgFileName][
"nbNotIgnoredPixels"] = np.count_nonzero(notIgnoredPixels)
perImageStats[predictionImgFileName][
"nbCorrectPixels"] = np.count_nonzero(erroneousPixels)
return nbPixels
def evaluatePair_image(predictionNp, groundTruthNp, confMatrix, instanceStats,
perImageStats, i_im, args):
# Loading all resources for evaluation.
predictionNp = np.asarray(predictionNp).astype('uint8')
groundTruthNp = np.asarray(groundTruthNp).astype('uint8')
predictionImg = Image.fromarray(predictionNp)
predictionNp = np.array(predictionImg)
groundTruthImg = Image.fromarray(groundTruthNp)
groundTruthNp = np.array(groundTruthImg)
imgWidth = predictionImg.size[0]
imgHeight = predictionImg.size[1]
nbPixels = imgWidth * imgHeight
# Evaluate images
if (CSUPPORT):
# using cython
confMatrix = addToConfusionMatrix.cEvaluatePair(
predictionNp, groundTruthNp, confMatrix, args.evalLabels)
else:
# the slower python way
for (groundTruthImgPixel,
predictionImgPixel) in izip(groundTruthImg.getdata(),
predictionImg.getdata()):
if (not groundTruthImgPixel in args.evalLabels):
printError(
"Unknown label with id {:}".format(groundTruthImgPixel))
confMatrix[groundTruthImgPixel][predictionImgPixel] += 1
if args.evalInstLevelScore:
# Generate category masks
categoryMasks = {}
for category in instanceStats["categories"]:
categoryMasks[category] = np.in1d(
predictionNp,
instanceStats["categories"][category]["labelIds"]).reshape(
predictionNp.shape)
instList = np.unique(instanceNp[instanceNp > 1000])
for instId in instList:
labelId = int(instId / 1000)
label = id2label[labelId]
if label.ignoreInEval:
continue
mask = instanceNp == instId
instSize = np.count_nonzero(mask)
tp = np.count_nonzero(predictionNp[mask] == labelId)
fn = instSize - tp
weight = args.avgClassSize[label.name] / float(instSize)
tpWeighted = float(tp) * weight
fnWeighted = float(fn) * weight
instanceStats["classes"][label.name]["tp"] += tp
instanceStats["classes"][label.name]["fn"] += fn
instanceStats["classes"][label.name]["tpWeighted"] += tpWeighted
instanceStats["classes"][label.name]["fnWeighted"] += fnWeighted
category = label.category
if category in instanceStats["categories"]:
catTp = 0
catTp = np.count_nonzero(
np.logical_and(mask, categoryMasks[category]))
catFn = instSize - catTp
catTpWeighted = float(catTp) * weight
catFnWeighted = float(catFn) * weight
instanceStats["categories"][category]["tp"] += catTp
instanceStats["categories"][category]["fn"] += catFn
instanceStats["categories"][category][
"tpWeighted"] += catTpWeighted
instanceStats["categories"][category][
"fnWeighted"] += catFnWeighted
if args.evalPixelAccuracy:
notIgnoredLabels = [
l for l in args.evalLabels if not id2label[l].ignoreInEval
]
notIgnoredPixels = np.in1d(groundTruthNp,
notIgnoredLabels,
invert=True).reshape(groundTruthNp.shape)
erroneousPixels = np.logical_and(notIgnoredPixels,
(predictionNp != groundTruthNp))
perImageStats[i_im] = {}
perImageStats[i_im]["nbNotIgnoredPixels"] = np.count_nonzero(
notIgnoredPixels)
perImageStats[i_im]["nbCorrectPixels"] = np.count_nonzero(
erroneousPixels)
return nbPixels
