def detection_metrics(coco_gt, coco_dt, params): def calk_cond_mean(s, area, cat_id=-1, iouThr="mean", maxDet=-1): p = coco_eval.params s = s[:, :, list(p.areaRngLbl).index(area), p.maxDets.index(maxDet)] if cat_id != -1: s = s[:, p.catIds.index(cat_id)] if iouThr != "mean": s = s[list(p.iouThrs).index(iouThr)] valid = s > -1 return np.mean(s[valid]) if valid.any() else -1 def AP(area, cat_id=-1, iouThr=None, maxDet=-1): s = coco_eval.eval['precision'].mean(axis=1) return calk_cond_mean(s, area, cat_id, iouThr, maxDet) def AR(area, cat_id=-1, iouThr=None, maxDet=-1): s = coco_eval.eval['recall'] return calk_cond_mean(s, area, cat_id, iouThr, maxDet) def pr_curve(area, cat_id, iouThr, maxDet): p = coco_eval.params recall = p.recThrs ti = list(p.iouThrs).index(iouThr) ki = list(p.catIds).index(cat_id) ai = list(p.areaRngLbl).index(area) di = list(p.maxDets).index(maxDet) precision = coco_eval.eval['precision'][ti, :, ki, ai, di] return recall, precision coco_eval = COCOeval(coco_gt, coco_dt, params.iouType) coco_eval.params = params coco_eval.evaluate() coco_eval.accumulate() metrics = [] p = coco_eval.params for cat_id in p.catIds: for area in p.areaRngLbl: for maxDet in p.maxDets: for iouThr in p.iouThrs: ap = AP(area, cat_id, iouThr, maxDet) ar = AR(area, cat_id, iouThr, maxDet) recall, precision = pr_curve(area, cat_id, iouThr, maxDet) metrics.append({ "class": p.id_to_class[cat_id], "area": area, "maxDet": maxDet, "iouThr": iouThr, "AP": ap, "AR": ar, "recall": list(recall), "precision": list(precision) }) return pd.DataFrame(metrics)
def run_eval_debug(self, results, save_dir): self.save_results(results, save_dir) save_path = "cache/fail/" coco_dets = self.coco.loadRes('{}/results.json'.format(save_dir)) coco_eval = COCOeval(self.coco, coco_dets, "bbox") p = coco_eval.params p.imgIds = list(np.unique(p.imgIds)) p.maxDets = sorted(p.maxDets) coco_eval.params = p coco_eval._prepare() catIds = [-1] computeIoU = coco_eval.computeIoU coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \ for imgId in p.imgIds for catId in catIds} maxDet = p.maxDets[-1] for imgId in p.imgIds: img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/" fullImgId = str(imgId).zfill(11) img_path = os.path.join(img_path, fullImgId + '.jpg') img = cv2.imread(img_path) for catId in catIds: # for i, areaRng in enumerate(p.areaRng): areaRng = p.areaRng[0] print("areaRng: " + areaRng) result = coco_eval.getImageFailCase(imgId, catId, areaRng, maxDet) gtIds = result['gtIds'] dtIds = result['dtIds'] gtFailIds = gtIds[result['gtfail'][0]] dtFailIds = dtIds[result['dtfail'][0]] gts = [ _ for cId in p.catIds for _ in coco_eval._gts[imgId, cId] ] dts = [ _ for cId in p.catIds for _ in coco_eval._dts[imgId, cId] ] gts = [g['bbox'] for g in gts] dts = [d['bbox'] for d in dts] pdb.set_trace() c = [255, 0, 0] for dt in dts: bbox = np.array(dt, dtype=np.int32) cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]), c, 2) c = [0, 255, 0] for gt in gts: bbox = np.array(gt, dtype=np.int32) cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]), c, 2) # saveImage save_path = os.path.join(save_path, fullImgId + "_" + str(i)) cv2.imwrite(save_path, img)
def _coco_eval(self): from .coco_wrapper import COCOWrapper coco_ground_truths = COCOWrapper.convert(self.targets, 'gt') coco_predictions = COCOWrapper.convert(self.predictions, 'prediction') coco_eval = COCOeval(coco_ground_truths, coco_predictions, 'bbox') self.coco_eval_params.catIds = coco_eval.params.catIds self.coco_eval_params.imgIds = coco_eval.params.imgIds coco_eval.params = self.coco_eval_params coco_eval.evaluate() coco_eval.accumulate() return coco_eval
def evaluate(): cocoGt = COCO('annotations.json') cocoDt = cocoGt.loadRes('detections.json') cocoEval = COCOeval(cocoGt, cocoDt, 'bbox') cocoEval.evaluate() p = cocoEval.params p.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .01) + 1, endpoint=True) cocoEval.params = p cocoEval.accumulate() cocoEval.summarize() print(cocoEval.eval['precision']) print(cocoEval.eval['precision'].shape) print(cocoEval.eval['recall'].shape)
def evaluate(iou_type_evaluator: COCOeval) -> Tuple: """ Run per image evaluation on given images and store results (a list of dict) in self.evalImgs :return: None """ p = iou_type_evaluator.params # add backward compatibility if useSegm is specified in params if p.useSegm is not None: p.iouType = "segm" if p.useSegm == 1 else "bbox" print( f"useSegm (deprecated) is not None. Running {p.iouType} evaluation" ) # print('Evaluate annotation type *{}*'.format(p.iouType)) p.imgIds = list(numpy.unique(p.imgIds)) if p.useCats: p.catIds = list(numpy.unique(p.catIds)) p.maxDets = sorted(p.maxDets) iou_type_evaluator.params = p iou_type_evaluator._prepare() # loop through images, area range, max detection number cat_ids = p.catIds if p.useCats else [-1] compute_iou = None if p.iouType == "segm" or p.iouType == "bbox": compute_iou = iou_type_evaluator.computeIoU elif p.iouType == "keypoints": compute_iou = iou_type_evaluator.computeOks iou_type_evaluator.ious = {(imgId, catId): compute_iou(imgId, catId) for imgId in p.imgIds for catId in cat_ids} evaluate_img = iou_type_evaluator.evaluateImg max_det = p.maxDets[-1] eval_imgs = [ evaluate_img(img_id, cat_id, area_rng, max_det) for cat_id in cat_ids for area_rng in p.areaRng for img_id in p.imgIds ] eval_imgs = numpy.asarray( eval_imgs ).reshape( # this is NOT in the pycocotools code, but could be done outside len(cat_ids), len(p.areaRng), len(p.imgIds)) iou_type_evaluator._paramsEval = copy.deepcopy(iou_type_evaluator.params) return p.imgIds, eval_imgs
'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] annot_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/annotations/instances_val2017.json" coco = coco.COCO(annot_path) save_dir = "/home/user/home/user/Xinyuan/work/CenterNet-1/exp/ctdet/coco_hg/" coco_dets = coco.loadRes('{}/results.json'.format(save_dir)) coco_eval = COCOeval(coco, coco_dets, "bbox") print(dir(coco_eval)) save_path = "src/cache/fail/" coco_dets = coco.loadRes('{}/results.json'.format(save_dir)) coco_eval = COCOeval(coco, coco_dets, "bbox") p = coco_eval.params p.imgIds = list(np.unique(p.imgIds)) p.maxDets = sorted(p.maxDets) coco_eval.params = p coco_eval._prepare() cat_dict = coco.cats # print(cat_dict) catIds = coco_eval.params.catIds computeIoU = coco_eval.computeIoU coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \ for imgId in p.imgIds for catId in catIds} maxDet = p.maxDets[-1] for imgId in p.imgIds: img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/" fullImgId = str(imgId).zfill(12) img_path = os.path.join(img_path, fullImgId + '.jpg') print(img_path) img = cv2.imread(img_path)