def detection_metrics(coco_gt, coco_dt, params):
def calk_cond_mean(s, area, cat_id=-1, iouThr="mean", maxDet=-1):
p = coco_eval.params
s = s[:, :, list(p.areaRngLbl).index(area), p.maxDets.index(maxDet)]
if cat_id != -1:
s = s[:, p.catIds.index(cat_id)]
if iouThr != "mean":
s = s[list(p.iouThrs).index(iouThr)]
valid = s > -1
return np.mean(s[valid]) if valid.any() else -1
def AP(area, cat_id=-1, iouThr=None, maxDet=-1):
s = coco_eval.eval['precision'].mean(axis=1)
return calk_cond_mean(s, area, cat_id, iouThr, maxDet)
def AR(area, cat_id=-1, iouThr=None, maxDet=-1):
s = coco_eval.eval['recall']
return calk_cond_mean(s, area, cat_id, iouThr, maxDet)
def pr_curve(area, cat_id, iouThr, maxDet):
p = coco_eval.params
recall = p.recThrs
ti = list(p.iouThrs).index(iouThr)
ki = list(p.catIds).index(cat_id)
ai = list(p.areaRngLbl).index(area)
di = list(p.maxDets).index(maxDet)
precision = coco_eval.eval['precision'][ti, :, ki, ai, di]
return recall, precision
coco_eval = COCOeval(coco_gt, coco_dt, params.iouType)
coco_eval.params = params
coco_eval.evaluate()
coco_eval.accumulate()
metrics = []
p = coco_eval.params
for cat_id in p.catIds:
for area in p.areaRngLbl:
for maxDet in p.maxDets:
for iouThr in p.iouThrs:
ap = AP(area, cat_id, iouThr, maxDet)
ar = AR(area, cat_id, iouThr, maxDet)
recall, precision = pr_curve(area, cat_id, iouThr, maxDet)
metrics.append({
"class": p.id_to_class[cat_id],
"area": area,
"maxDet": maxDet,
"iouThr": iouThr,
"AP": ap,
"AR": ar,
"recall": list(recall),
"precision": list(precision)
})
return pd.DataFrame(metrics)
def run_eval_debug(self, results, save_dir):
self.save_results(results, save_dir)
save_path = "cache/fail/"
coco_dets = self.coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(self.coco, coco_dets, "bbox")
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
p.maxDets = sorted(p.maxDets)
coco_eval.params = p
coco_eval._prepare()
catIds = [-1]
computeIoU = coco_eval.computeIoU
coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \
for imgId in p.imgIds
for catId in catIds}
maxDet = p.maxDets[-1]
for imgId in p.imgIds:
img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/"
fullImgId = str(imgId).zfill(11)
img_path = os.path.join(img_path, fullImgId + '.jpg')
img = cv2.imread(img_path)
for catId in catIds:
# for i, areaRng in enumerate(p.areaRng):
areaRng = p.areaRng[0]
print("areaRng: " + areaRng)
result = coco_eval.getImageFailCase(imgId, catId, areaRng,
maxDet)
gtIds = result['gtIds']
dtIds = result['dtIds']
gtFailIds = gtIds[result['gtfail'][0]]
dtFailIds = dtIds[result['dtfail'][0]]
gts = [
_ for cId in p.catIds for _ in coco_eval._gts[imgId, cId]
]
dts = [
_ for cId in p.catIds for _ in coco_eval._dts[imgId, cId]
]
gts = [g['bbox'] for g in gts]
dts = [d['bbox'] for d in dts]
pdb.set_trace()
c = [255, 0, 0]
for dt in dts:
bbox = np.array(dt, dtype=np.int32)
cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
c, 2)
c = [0, 255, 0]
for gt in gts:
bbox = np.array(gt, dtype=np.int32)
cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]),
c, 2)
# saveImage
save_path = os.path.join(save_path, fullImgId + "_" + str(i))
cv2.imwrite(save_path, img)
def _coco_eval(self):
from .coco_wrapper import COCOWrapper
coco_ground_truths = COCOWrapper.convert(self.targets, 'gt')
coco_predictions = COCOWrapper.convert(self.predictions, 'prediction')
coco_eval = COCOeval(coco_ground_truths, coco_predictions, 'bbox')
self.coco_eval_params.catIds = coco_eval.params.catIds
self.coco_eval_params.imgIds = coco_eval.params.imgIds
coco_eval.params = self.coco_eval_params
coco_eval.evaluate()
coco_eval.accumulate()
return coco_eval
def evaluate():
cocoGt = COCO('annotations.json')
cocoDt = cocoGt.loadRes('detections.json')
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.evaluate()
p = cocoEval.params
p.iouThrs = np.linspace(.5,
0.95,
np.round((0.95 - .5) / .01) + 1,
endpoint=True)
cocoEval.params = p
cocoEval.accumulate()
cocoEval.summarize()
print(cocoEval.eval['precision'])
print(cocoEval.eval['precision'].shape)
print(cocoEval.eval['recall'].shape)
def evaluate(iou_type_evaluator: COCOeval) -> Tuple:
"""
Run per image evaluation on given images and store results (a list of dict) in self.evalImgs
:return: None
"""
p = iou_type_evaluator.params
# add backward compatibility if useSegm is specified in params
if p.useSegm is not None:
p.iouType = "segm" if p.useSegm == 1 else "bbox"
print(
f"useSegm (deprecated) is not None. Running {p.iouType} evaluation"
)
# print('Evaluate annotation type *{}*'.format(p.iouType))
p.imgIds = list(numpy.unique(p.imgIds))
if p.useCats:
p.catIds = list(numpy.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
iou_type_evaluator.params = p
iou_type_evaluator._prepare()
# loop through images, area range, max detection number
cat_ids = p.catIds if p.useCats else [-1]
compute_iou = None
if p.iouType == "segm" or p.iouType == "bbox":
compute_iou = iou_type_evaluator.computeIoU
elif p.iouType == "keypoints":
compute_iou = iou_type_evaluator.computeOks
iou_type_evaluator.ious = {(imgId, catId): compute_iou(imgId, catId)
for imgId in p.imgIds for catId in cat_ids}
evaluate_img = iou_type_evaluator.evaluateImg
max_det = p.maxDets[-1]
eval_imgs = [
evaluate_img(img_id, cat_id, area_rng, max_det) for cat_id in cat_ids
for area_rng in p.areaRng for img_id in p.imgIds
]
eval_imgs = numpy.asarray(
eval_imgs
).reshape( # this is NOT in the pycocotools code, but could be done outside
len(cat_ids), len(p.areaRng), len(p.imgIds))
iou_type_evaluator._paramsEval = copy.deepcopy(iou_type_evaluator.params)
return p.imgIds, eval_imgs
'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',
'hair drier', 'toothbrush'
]
annot_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/annotations/instances_val2017.json"
coco = coco.COCO(annot_path)
save_dir = "/home/user/home/user/Xinyuan/work/CenterNet-1/exp/ctdet/coco_hg/"
coco_dets = coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(coco, coco_dets, "bbox")
print(dir(coco_eval))
save_path = "src/cache/fail/"
coco_dets = coco.loadRes('{}/results.json'.format(save_dir))
coco_eval = COCOeval(coco, coco_dets, "bbox")
p = coco_eval.params
p.imgIds = list(np.unique(p.imgIds))
p.maxDets = sorted(p.maxDets)
coco_eval.params = p
coco_eval._prepare()
cat_dict = coco.cats
# print(cat_dict)
catIds = coco_eval.params.catIds
computeIoU = coco_eval.computeIoU
coco_eval.ious = {(imgId, catId): coco_eval.computeIoU(imgId, catId) \
for imgId in p.imgIds
for catId in catIds}
maxDet = p.maxDets[-1]
for imgId in p.imgIds:
img_path = "/home/user/home/user/Xinyuan/work/CenterNet-1/data/coco/val2017/"
fullImgId = str(imgId).zfill(12)
img_path = os.path.join(img_path, fullImgId + '.jpg')
print(img_path)
img = cv2.imread(img_path)
