def eval_ccf(db, results, class_subset=None, per_class=False, iou_type='bbox'): # ccf means CoCo Format if isinstance(results, str): if results.endswith('.pkl'): results = pickle.load(open(results, 'rb')) else: results = json.load(open(results, 'r')) # if iou_type == 'segm': # for i in range(len(results)): # results[i]['segmentation']['counts'] = results[i]['segmentation']['counts'].decode() # # det.pop('bbox', None) results = db.loadRes(results) cocoEval = COCOeval(db, results, iou_type) if class_subset is not None: cocoEval.params.catIds = class_subset def summarize_per_class(self): ''' Compute and display summary metrics for evaluation results. Note this functin can *only* be applied on the default parameter setting ''' def _summarize(ap=1, iouThr=None, areaRng='all', maxDets=100): p = self.params iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' titleStr = 'Average Precision' if ap == 1 else 'Average Recall' typeStr = '(AP)' if ap == 1 else '(AR)' iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ if iouThr is None else '{:0.2f}'.format(iouThr) aind = [ i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng ] mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] if ap == 1: # dimension of precision: [TxRxKxAxM] s = self.eval['precision'] # IoU if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] s = s[:, :, :, aind, mind] else: # dimension of recall: [TxKxAxM] s = self.eval['recall'] if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] s = s[:, :, aind, mind] if len(s[s > -1]) == 0: mean_s = -1 else: # print(s.shape) for k in range(s.shape[2]): print(np.mean(s[:, :, k])) mean_s = np.mean(s[s > -1]) print( iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) return mean_s def _summarizeDets(): stats = np.zeros((12, )) stats[0] = _summarize(1) # stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) # stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2]) # stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2]) # stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2]) # stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2]) # stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) # stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) # stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) # stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2]) # stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2]) # stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) return stats if not self.eval: raise Exception('Please run accumulate() first') iouType = self.params.iouType if iouType == 'segm' or iouType == 'bbox': summarize = _summarizeDets elif iouType == 'keypoints': raise NotImplementedError self.stats = summarize() def __str__(self): self.summarize() # r = 8.417032065165149 # ArgoVerse (1920x1200) / mean COCO # for rng in cocoEval.params.areaRng: # rng[0] *= r # rng[1] *= r cocoEval.evaluate() cocoEval.accumulate() if per_class: cocoEval.summarize_per_class = MethodType(summarize_per_class, cocoEval) cocoEval.summarize_per_class() else: cocoEval.summarize() return { 'eval': cocoEval.eval, 'stats': cocoEval.stats, }