def defaultHitStrategy(origpur, foundLabels, foundMap, tres):
numfound = len(foundLabels)
if numfound==0:
if origpur.ptype==cvac.PurposeType.POSITIVE:
tres.fn += 1
else:
tres.tn += 1
return tres
foundPurposes = []
'''
Get one count each of each purpose found. We use this to make sure that we
only return a result for each purpose and no more. If more than one result for each
purpose is in the foundLabels, then only the first one is counted.
'''
for lbl in foundLabels:
labelText = easy.getLabelText( lbl.lab, guess=False )
if labelText in foundMap:
foundPurpose = foundMap[labelText]
if foundPurpose not in foundPurposes:
foundPurposes.append(foundPurpose)
else:
print("warning: Label " + labelText + " not in foundMap can't compute evaluation.")
for lbl in foundLabels:
labelText = easy.getLabelText( lbl.lab, guess=False )
if labelText in foundMap:
foundPurpose = foundMap[labelText]
'''We only want to count a purpose once
so remove it from the foundPurposes list '''
if foundPurpose in foundPurposes:
foundPurposes.remove(foundPurpose)
if origpur.ptype == cvac.PurposeType.POSITIVE:
if foundPurpose.ptype == cvac.PurposeType.POSITIVE:
tres.tp += 1
else:
tres.fn += 1
else:
if foundPurpose.ptype == cvac.PurposeType.POSITIVE:
# This defaultHitStrategy counts every false positive
tres.fp += 1
else:
tres.tn += 1
return tres
def getConfusionTable( results, foundMap, origMap=None, origSet=None,
multipleHitStrategy=MultipleHitStrategy.CountEveryFalsePositive ):
'''Determine true and false positives and negatives based on
the purpose of original and found labels.
origMap maps the relative file path of every label to the assigned purpose.
The origMap can be constructed from the original RunSet if it
contained purposes.
Returns TestReult, nores'''
if not origMap and not origSet:
raise RuntimeError("need either origMap or origSet")
if not foundMap:
raise RuntimeError("need a foundMap")
if not verifyFoundMap(foundMap):
raise RuntimeError("Invalid found map")
if not origMap:
origMap = {}
for plist in origSet.purposedLists:
assert( isinstance(plist, cvac.PurposedLabelableSeq) )
for sample in plist.labeledArtifacts:
if plist.pur.ptype != cvac.PurposeType.POSITIVE \
and plist.pur.ptype != cvac.PurposeType.NEGATIVE:
raise RuntimeError("Non pos or neg purpose in runset")
origMap[ getRelativePath(sample) ] = plist.pur
# compute the number of samples in the origSet that was not evaluated
nores = 0
if origSet:
origSize = len( asList( origSet ) )
nores = origSize - len(results)
else:
print("warning: Not able to determine samples not evaluated")
tres = TestResult()
for res in results:
foundPurposes = []
for lbl in res.foundLabels:
labelText = easy.getLabelText( lbl.lab, guess=False )
if labelText in foundMap:
foundPurpose = foundMap[labelText]
foundPurposes.append(foundPurpose)
else:
print("warning: Label " + labelText + " not in foundMap can't compute evaluation.")
numfound = len(res.foundLabels)
origpur = origMap[ getRelativePath(res.original) ]
if numfound==0:
if origpur.ptype==cvac.PurposeType.POSITIVE:
tres.fn += 1
else:
tres.tn += 1
else:
# We found multiple things so look at multiple hit strategy to see how to count
foundFalsePos = False
for lbl in res.foundLabels:
labelText = easy.getLabelText( lbl.lab, guess=False )
if labelText in foundMap:
foundPurpose = foundMap[labelText]
'''We only want to count a purpose once
so remove it from the foundPurposes list '''
if foundPurpose in foundPurposes:
foundPurposes.remove(foundPurpose)
if origpur.ptype == cvac.PurposeType.POSITIVE:
if foundPurpose.ptype == cvac.PurposeType.POSITIVE:
tres.tp += 1
else:
tres.fn += 1
else:
if foundPurpose.ptype == cvac.PurposeType.POSITIVE:
if multipleHitStrategy == MultipleHitStrategy.CountEveryFalsePositive:
tres.fp += 1
elif foundFalsePos == False:
tres.fp += 1
foundFalsePos = True
else:
tres.tn += 1
print('{0}, nores: {1}'.format(tres, nores))
return tres, nores