def evaluate_intuitive(gt, pd, overlapThreshold=0.5, outputMatching=False):
"""
Evaluate the prediction against the ground truth using intuitive metric.
In an overlapping set, match ground truth labels to predictions with the same label.
(e.g., if gt = [person, dog] and pd = [dog, person] in an overlapping set, match the labels to each other
:param gt: ground truth dict as {frame: {label: [xmin, ymin, xmax, ymax]}}
:param pd: prediction dict as {frame: {label: ([xmin, ymin, xmax, ymax], confidence)}}
:return: mAP metric on dataset
"""
## make assignments of bounding boxes based on overlap
# find all bounding boxes with overlap at desired threshold
outerDic = {}
for frame in gt:
matchedDict = defaultdict(bool) # contains the predictions we've already matched
gtq = gt[frame].keys() # store keys as queue
innerDic = {}
while gtq:
gtLabel = gtq.pop()
gbox = gt[frame][gtLabel]
pMatchList = []
for pdLabel in pd[frame]:
pbox = pd[frame][pdLabel][0]
ol = calc_overlap(pbox, gbox)
if ol > overlapThreshold:
pMatchList.append([pdLabel, pd[frame][pdLabel]])
if pMatchList:
# choose the bounding box with highest confidence as prediction
pMatchList.sort(key=lambda match: match[1][1], reverse=True)
pMatch = pMatchList[0][0]
#pMatch = pMatchList[np.argmax([match[1][1] for match in pMatchList])]
if matchedDict[pMatch]:
## resolve conflict between 2 ground truth values wanting the same prediction
# find original ground truth value matching this prediction
gPrev = None
for gp, pp in innerDic.items():
if pp == pMatch:
gPrev = gp
if not gPrev:
raise ValueError('gPrev should not be None if a match occurred')
# check which match is better
if pMatch in innerDic and pMatch == innerDic[pMatch]:
# if current label matches current prediction, then keep and take next best c-score
if len(pMatchList) > 1:
innerDic[gtLabel] = pMatchList[1][0]
matchedDict[pMatchList[1][0]] = True
# if no next best prediction, then don't make any new match
else:
# form list of possible other matches
gMatchList = []
for gMatch in gt[frame]:
gboxn = gt[frame][gMatch]
pboxn = pd[frame][pMatch][0]
ol = calc_overlap(pboxn, gboxn)
if ol > overlapThreshold:
gMatchList.append(gMatch)
# check if prediction matches any ground truth label in possible set
for g in gMatchList:
if g == pMatch: # note this will only be true at most once by uniqueness of keys
innerDic.pop(gPrev)
gtq.append(gPrev)
innerDic[g] = pMatch
# note matchedDict[pMatch] is already true for this prediction
# if not, then choose one with greater overlap
if not matchedDict[pMatch]:
olp = calc_overlap(gt[frame][gPrev], pd[frame][pMatch][0])
oln = calc_overlap(gt[frame][gtLabel], pd[frame][pMatch][0])
if olp > oln:
# keep current matching and take next best c-score
if len(pMatchList) > 1:
innerDic[gtLabel] = pMatchList[1][0]
matchedDict[pMatchList[1][0]] = True
else:
innerDic.pop(gPrev)
gtq.append(gPrev)
innerDic[gtLabel] = pMatch
else:
matchedDict[pMatch] = True
innerDic[gtLabel] = pMatch
for gk in gt[frame]:
if gk not in innerDic:
innerDic[gk] = ''
outerDic[frame] = innerDic
# outerDic format {frame: {gtlabel: pdlabel}}
# calculate mAP based on matched labels
outMap = calc_mAP_from_dict(outerDic)
# return desired output
if outputMatching:
return outMap, outerDic
else:
return outMap
def evaluate_mAP(gt, pd, k=2, overlapThreshold=0.5):
"""
Evaluate the prediction against the ground truth using mAP
The idea is to find all sets of overlapping boxes, then evaluate the mean average precision
:param gt: ground truth list of lists where each element is (label, [xmin, ymin, xmax, ymax]);
rows refer to frames and columns to objects within frames
:param pd: prediction list of lists where each element is (label, [xmin, ymin, xmax, ymax], confidence);
rows refer to frames and columns to objects within frames
:return: mAP metric on dataset
"""
if isinstance(gt, Mapping) and isinstance(pd, Mapping):
gtList = [d.items() for d in gt.values()]
pdList = [list((key, d[key][0], d[key][1]) for key in d.keys()) for d in pd.values()]
elif isinstance(gt, Sequence) and isinstance(pd, Sequence):
gtList = gt
pdList = pd
else:
raise TypeError('Inputs should be lists of lists where each element is (label, [xmin, ymin, xmax, ymax], confidence);'
'rows refer to frames and columns to objects within frames')
## make assignments of bounding boxes based on overlap
# find all bounding boxes with overlap at desired threshold
gMatchListOuter = []
pMatchListOuter = []
for i, frame in enumerate(gtList):
gtq = frame[:]
pdq = pdList[i][:]
while gtq:
# check all overlapping sets containing at least one ground truth value
gtCurrent = gtq.pop()
gtLabel = gtCurrent[0]
gtbox = gtCurrent[1]
pMatchList = []
for j1, pCurrent in enumerate(pdList[i]):
pLabel = pCurrent[0]
pbox = pCurrent[1]
ol = calc_overlap(pbox, gtbox)
if ol > overlapThreshold:
pMatchList.append(pCurrent)
if pCurrent in pdq:
pdq.remove(pCurrent)
pMatchList.sort(key=lambda match: match[2], reverse=True)
pMatchList = [match[0] for match in pMatchList]
gMatchList = [gtLabel]
for j2, gCurrent in enumerate(gtq[:]):
gLabel = gCurrent[0]
gbox = gCurrent[1]
ol = calc_overlap(gtbox, gbox)
if ol > overlapThreshold:
# part of the same set of overlapping bounding boxes
gMatchList.append(gLabel)
gtq.remove(gCurrent)
gMatchListOuter.append(gMatchList)
pMatchListOuter.append(pMatchList)
while pdq:
# check remaining sets containing no ground truth value
pdCurrent = pdq.pop()
pdLabel = pdCurrent[0]
pdbox = pdCurrent[1]
pMatchList = [pdLabel]
for j, pCurrent in enumerate(pdq[:]):
pLabel = pCurrent[0]
pbox = pCurrent[1]
ol = calc_overlap(pdbox, pbox)
if ol > overlapThreshold:
# part of the same set of overlapping bounding boxes
pMatchList.append(pLabel)
pdq.remove(pCurrent)
gMatchListOuter.append([])
pMatchListOuter.append(pMatchList)
# calculate mAP based on overlapping regions
return calc_mAP(gMatchListOuter, pMatchListOuter, k=k)
def produce(self, ip):
# expected output format
# {
# 'object_1': {
# 0: { # Frame index
# 'labels': ['Label 1', 'Label 3'] # Optional Labels
# 'block' : [0, 0, 200, 300] # Optional 'spatial' block
# }, ...
# },
# 'object_2: [ ... ]
# }
outDict = {}
classCounter = defaultdict(int) # track how many instances of each class have been created to ensure unique naming
pClassDict = {}
cClassDict = {}
cInst = {} # keep track of current detection instance
# helper function to create new current instance
def make_cInst(objClass, block, conf, labels):
cInst['cls'] = objClass
outBlock = [float(block[0]), float(block[1]), float(block[2]), float(block[3])]
cInst['block'] = outBlock
cInst['conf'] = float(conf)
cInst['labels'] = labels
return cInst
# helper function to add the current instance to the current class dictionary or output class dictionary
def add_cInst(container, cInst, frame=None):
if frame is None:
container[cInst['cls']] = {'block': cInst['block'], 'conf': cInst['conf'], 'labels': cInst['labels']}
else:
try:
container[cInst['cls']][frame] = {'block': cInst['block'], 'conf': cInst['conf'], 'labels': cInst['labels']}
except KeyError:
container[cInst['cls']] = {frame: {'block': cInst['block'], 'conf': cInst['conf'], 'labels': cInst['labels']}}
return container
# RCNN output - frame: obj: block list
for frameNum, frameDets in enumerate(ip):
# handle first frame special case - all detections create new objects
if frameNum == 0:
for objClass in frameDets:
for objInst in frameDets[objClass]:
classCounter[objClass] += 1
objID = objClass + '_' + str(classCounter[objClass])
cInst = make_cInst(objID, objInst[:4], objInst[-1], [cfg.dataset2label(objClass)])
add_cInst(cClassDict, cInst)
add_cInst(outDict, cInst, frame=1)
else:
# handle general case
for objClass in frameDets:
for objInst in frameDets[objClass]:
cInst = make_cInst(objClass, objInst[:4], objInst[-1], [cfg.dataset2label(objClass)])
pOverlap = 0
for pObjID in pClassDict:
pClsRoot = pObjID.split('_')[0]
if pClsRoot == objClass:
ol = mm.calc_overlap(pClassDict[pObjID]['block'], cInst['block'])
if ol > 0.1 and pOverlap == 0:
# first overlap
pOverlap = 1
cInst = make_cInst(pObjID, cInst['block'], cInst['conf'], cInst['labels'])
elif ol > 0.1 and pOverlap == 1:
# there has already been an overlap
pOverlap = -1
if pOverlap == 1:
# there was only one overlap for this instance in this direction
if cInst['cls'] in cClassDict:
# overlap in the other direction - seperate into 2 objects
cClsRoot = cInst['cls'].split('_')[0]
# get previous instance
pDict = cClassDict.pop(cInst['cls'])
outDict.pop(cInst['cls'])
# add new instance
classCounter[cClsRoot] += 1
objID = objClass + '_' + str(classCounter[cClsRoot])
cInst = make_cInst(objID, cInst['block'], cInst['conf'], cInst['labels'])
add_cInst(cClassDict, cInst)
add_cInst(outDict, cInst, frame=frameNum+1)
# update previous instance
classCounter[cClsRoot] += 1
objID = objClass + '_' + str(classCounter[cClsRoot])
cInst = make_cInst(objID, pDict['block'], pDict['conf'], pDict['labels'])
add_cInst(cClassDict, cInst)
add_cInst(outDict, cInst, frame=frameNum+1)
else:
# no overlap in the other direction - add cInst as is
add_cInst(cClassDict, cInst)
add_cInst(outDict, cInst, frame=frameNum+1)
else:
# no overlap or multi overlap - create new object
classCounter[objClass] += 1
objID = objClass + '_' + str(classCounter[objClass])
cInst = make_cInst(objID, cInst['block'], cInst['conf'], cInst['labels'])
add_cInst(cClassDict, cInst)
add_cInst(outDict, cInst, frame=frameNum+1)
# set up next iteration
pClassDict = cClassDict
cClassDict = {}
return outDict