def eval_ccf(db, results, class_subset=None, per_class=False, iou_type='bbox'):
# ccf means CoCo Format
if isinstance(results, str):
if results.endswith('.pkl'):
results = pickle.load(open(results, 'rb'))
else:
results = json.load(open(results, 'r'))
# if iou_type == 'segm':
# for i in range(len(results)):
# results[i]['segmentation']['counts'] = results[i]['segmentation']['counts'].decode()
# # det.pop('bbox', None)
results = db.loadRes(results)
cocoEval = COCOeval(db, results, iou_type)
if class_subset is not None:
cocoEval.params.catIds = class_subset
def summarize_per_class(self):
'''
Compute and display summary metrics for evaluation results.
Note this functin can *only* be applied on the default parameter setting
'''
def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100):
p = self.params
iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
typeStr = '(AP)' if ap == 1 else '(AR)'
iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \
if iouThr is None else '{:0.2f}'.format(iouThr)
aind = [
i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng
]
mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
if ap == 1:
# dimension of precision: [TxRxKxAxM]
s = self.eval['precision']
# IoU
if iouThr is not None:
t = np.where(iouThr == p.iouThrs)[0]
s = s[t]
s = s[:, :, :, aind, mind]
else:
# dimension of recall: [TxKxAxM]
s = self.eval['recall']
if iouThr is not None:
t = np.where(iouThr == p.iouThrs)[0]
s = s[t]
s = s[:, :, aind, mind]
if len(s[s > -1]) == 0:
mean_s = -1
else:
# print(s.shape)
for k in range(s.shape[2]):
print(np.mean(s[:, :, k]))
mean_s = np.mean(s[s > -1])
print(
iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets,
mean_s))
return mean_s
def _summarizeDets():
stats = np.zeros((12, ))
stats[0] = _summarize(1)
# stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2])
# stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2])
# stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2])
# stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2])
# stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2])
# stats[6] = _summarize(0, maxDets=self.params.maxDets[0])
# stats[7] = _summarize(0, maxDets=self.params.maxDets[1])
# stats[8] = _summarize(0, maxDets=self.params.maxDets[2])
# stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2])
# stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2])
# stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2])
return stats
if not self.eval:
raise Exception('Please run accumulate() first')
iouType = self.params.iouType
if iouType == 'segm' or iouType == 'bbox':
summarize = _summarizeDets
elif iouType == 'keypoints':
raise NotImplementedError
self.stats = summarize()
def __str__(self):
self.summarize()
# r = 8.417032065165149 # ArgoVerse (1920x1200) / mean COCO
# for rng in cocoEval.params.areaRng:
# rng[0] *= r
# rng[1] *= r
cocoEval.evaluate()
cocoEval.accumulate()
if per_class:
cocoEval.summarize_per_class = MethodType(summarize_per_class,
cocoEval)
cocoEval.summarize_per_class()
else:
cocoEval.summarize()
return {
'eval': cocoEval.eval,
'stats': cocoEval.stats,
}
