def _do_python_keypoint_eval(self, res_file): """Keypoint evaluation using COCOAPI.""" stats_names = [ 'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] with open(res_file, 'r') as file: res_json = json.load(file) if not res_json: info_str = list(zip(stats_names, [ 0, ] * len(stats_names))) return info_str coco_det = self.coco.loadRes(res_file) coco_eval = COCOeval(self.coco, coco_det, 'keypoints', self.sigmas) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() info_str = list(zip(stats_names, coco_eval.stats)) return info_str
def _do_python_keypoint_eval(self, res_file): """Keypoint evaluation using COCOAPI.""" coco_det = self.coco.loadRes(res_file) coco_eval = COCOeval(self.coco, coco_det, 'keypoints', self.sigmas) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats_names = [ 'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] info_str = list(zip(stats_names, coco_eval.stats)) return info_str
def _do_python_keypoint_eval(self, res_file): """Keypoint evaluation using COCOAPI.""" coco_dt = self.coco.loadRes(res_file) coco_eval = COCOeval(self.coco, coco_dt, 'keypoints') coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats_names = [ 'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] info_str = [] for ind, name in enumerate(stats_names): info_str.append((name, coco_eval.stats[ind])) return info_str
0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018, 0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031 ] sigmas_righthand = [ 0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018, 0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031 ] sigmas_wholebody = sigmas_body + sigmas_foot + sigmas_face + sigmas_lefthand + sigmas_righthand cocoGt = COCO(gt_file) cocoDt = cocoGt.loadRes(preds) cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_body', np.array(sigmas_body), use_area=True) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_foot', np.array(sigmas_foot), use_area=True) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def _do_python_keypoint_eval(self, res_file): """Keypoint evaluation using COCOAPI.""" coco_det = self.coco.loadRes(res_file) coco_eval = COCOeval(self.coco, coco_det, 'keypoints_body', np.array(self.sigmas_body), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_det, 'keypoints_foot', np.array(self.sigmas_foot), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_det, 'keypoints_face', np.array(self.sigmas_face), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_det, 'keypoints_lefthand', np.array(self.sigmas_lefthand), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_det, 'keypoints_righthand', np.array(self.sigmas_righthand), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_det, 'keypoints_wholebody', np.array(self.sigmas_wholebody), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats_names = [ 'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] info_str = list(zip(stats_names, coco_eval.stats)) return info_str
def cocoapi_eval(jsonfile, style, coco_gt=None, anno_file=None, max_dets=(100, 300, 1000), classwise=False, sigmas=None, use_area=True): """ Args: jsonfile (str): Evaluation json file, eg: bbox.json, mask.json. style (str): COCOeval style, can be `bbox` , `segm` , `proposal`, `keypoints` and `keypoints_crowd`. coco_gt (str): Whether to load COCOAPI through anno_file, eg: coco_gt = COCO(anno_file) anno_file (str): COCO annotations file. max_dets (tuple): COCO evaluation maxDets. classwise (bool): Whether per-category AP and draw P-R Curve or not. sigmas (nparray): keypoint labelling sigmas. use_area (bool): If gt annotations (eg. CrowdPose, AIC) do not have 'area', please set use_area=False. """ assert coco_gt != None or anno_file != None if style == 'keypoints_crowd': #please install xtcocotools==1.6 from xtcocotools.coco import COCO from xtcocotools.cocoeval import COCOeval else: from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval if coco_gt == None: coco_gt = COCO(anno_file) logger.info("Start evaluate...") coco_dt = coco_gt.loadRes(jsonfile) if style == 'proposal': coco_eval = COCOeval(coco_gt, coco_dt, 'bbox') coco_eval.params.useCats = 0 coco_eval.params.maxDets = list(max_dets) elif style == 'keypoints_crowd': coco_eval = COCOeval(coco_gt, coco_dt, style, sigmas, use_area) else: coco_eval = COCOeval(coco_gt, coco_dt, style) coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() if classwise: # Compute per-category AP and PR curve try: from terminaltables import AsciiTable except Exception as e: logger.error( 'terminaltables not found, plaese install terminaltables. ' 'for example: `pip install terminaltables`.') raise e precisions = coco_eval.eval['precision'] cat_ids = coco_gt.getCatIds() # precision: (iou, recall, cls, area range, max dets) assert len(cat_ids) == precisions.shape[2] results_per_category = [] for idx, catId in enumerate(cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image nm = coco_gt.loadCats(catId)[0] precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (str(nm["name"]), '{:0.3f}'.format(float(ap)))) pr_array = precisions[0, :, idx, 0, 2] recall_array = np.arange(0.0, 1.01, 0.01) draw_pr_curve(pr_array, recall_array, out_dir=style + '_pr_curve', file_name='{}_precision_recall_curve.jpg'.format( nm["name"])) num_columns = min(6, len(results_per_category) * 2) results_flatten = list(itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest( *[results_flatten[i::num_columns] for i in range(num_columns)]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) logger.info('Per-category of {} AP: \n{}'.format(style, table.table)) logger.info("per-category PR curve has output to {} folder.".format( style + '_pr_curve')) # flush coco evaluation result sys.stdout.flush() return coco_eval.stats
def _do_python_keypoint_eval(self, res_file): """Keypoint evaluation using COCOAPI.""" coco_det = self.coco.loadRes(res_file) cuts = np.cumsum([ 0, self.body_num, self.foot_num, self.face_num, self.left_hand_num, self.right_hand_num ]) coco_eval = COCOeval( self.coco, coco_det, 'keypoints_body', self.sigmas[cuts[0]:cuts[1]], use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval( self.coco, coco_det, 'keypoints_foot', self.sigmas[cuts[1]:cuts[2]], use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval( self.coco, coco_det, 'keypoints_face', self.sigmas[cuts[2]:cuts[3]], use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval( self.coco, coco_det, 'keypoints_lefthand', self.sigmas[cuts[3]:cuts[4]], use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval( self.coco, coco_det, 'keypoints_righthand', self.sigmas[cuts[4]:cuts[5]], use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval( self.coco, coco_det, 'keypoints_wholebody', self.sigmas, use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats_names = [ 'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] info_str = list(zip(stats_names, coco_eval.stats)) return info_str
def _stats(self, predictions=None, image_ids=None): # from pycocotools.cocoeval import COCOeval if predictions is None: predictions = self.predictions if image_ids is None: image_ids = self.image_ids coco_eval = self.coco.loadRes(predictions) for count in coco_eval.anns: ann_orig = copy.deepcopy(coco_eval.anns[count]) coco_eval.anns[count]["lefthand_kpts"] = ann_orig["keypoints"][91 * 3:112 * 3] coco_eval.anns[count]["righthand_kpts"] = ann_orig["keypoints"][112 * 3:133 * 3] coco_eval.anns[count]["face_kpts"] = ann_orig["keypoints"][23 * 3:91 * 3] coco_eval.anns[count]["foot_kpts"] = ann_orig["keypoints"][17 * 3:23 * 3] coco_eval.anns[count]["keypoints"] = ann_orig["keypoints"][0:17 * 3] coco_eval.anno_file = coco_eval.anns # xtpycocotools style self.coco.anno_file = self.coco.anns eval_body = COCOeval(self.coco, coco_eval, 'keypoints_body', np.array(self.sigmas_body), use_area=True) eval_body.evaluate() eval_body.accumulate() eval_body.summarize() eval_foot = COCOeval(self.coco, coco_eval, 'keypoints_foot', np.array(self.sigmas_foot), use_area=True) eval_foot.evaluate() eval_foot.accumulate() eval_foot.summarize() eval_face = COCOeval(self.coco, coco_eval, 'keypoints_face', np.array(self.sigmas_face), use_area=True) eval_face.evaluate() eval_face.accumulate() eval_face.summarize() eval_lh = COCOeval(self.coco, coco_eval, 'keypoints_lefthand', np.array(self.sigmas_lefthand), use_area=True) eval_lh.evaluate() eval_lh.accumulate() eval_lh.summarize() eval_rh = COCOeval(self.coco, coco_eval, 'keypoints_righthand', np.array(self.sigmas_righthand), use_area=True) eval_rh.evaluate() eval_rh.accumulate() eval_rh.summarize() eval_wb = COCOeval(self.coco, coco_eval, 'keypoints_wholebody', np.array(self.sigmas_wholebody), use_area=True) eval_wb.evaluate() eval_wb.accumulate() eval_wb.summarize() return eval_wb.stats
from xtcocotools.coco import COCO from xtcocotools.cocoeval import COCOeval import numpy as np gt_file = '../annotations/example_coco_val.json' preds = '../annotations/example_coco_preds.json' sigmas = np.array( [.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0 cocoGt = COCO(gt_file) cocoDt = cocoGt.loadRes(preds) cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints', sigmas, use_area=True) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()
def evaluate_mAP(res_file, ann_type='bbox', ann_file='./data/coco/annotations/person_keypoints_val2017.json', silence=True): """Evaluate mAP result for coco dataset. Parameters ---------- res_file: str Path to result json file. ann_type: str annotation type, including: `bbox`, `segm`, `keypoints`. ann_file: str Path to groundtruth file. silence: bool True: disable running log. """ class NullWriter(object): def write(self, arg): pass # ann_file = os.path.join('./data/coco/annotations/', ann_file) sigmas_body = [0.026, 0.025, 0.025, 0.035, 0.035, 0.079, 0.079, 0.072, 0.072, 0.062, 0.062, 0.107, 0.107, 0.087,0.087, 0.089, 0.089] sigmas_foot = [0.068, 0.066, 0.066, 0.092, 0.094, 0.094] sigmas_face = [0.042, 0.043, 0.044, 0.043, 0.040, 0.035, 0.031, 0.025, 0.020, 0.023, 0.029, 0.032, 0.037, 0.038, 0.043,0.041, 0.045, 0.013, 0.012, 0.011, 0.011, 0.012, 0.012, 0.011, 0.011, 0.013, 0.015, 0.009, 0.007, 0.007,0.007, 0.012, 0.009, 0.008, 0.016, 0.010, 0.017, 0.011, 0.009, 0.011, 0.009, 0.007, 0.013, 0.008, 0.011,0.012, 0.010, 0.034, 0.008, 0.008, 0.009, 0.008, 0.008, 0.007, 0.010, 0.008, 0.009, 0.009, 0.009, 0.007, 0.007, 0.008, 0.011, 0.008, 0.008, 0.008, 0.01, 0.008] sigmas_lefthand = [0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018, 0.024, 0.022, 0.026, 0.017,0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031] sigmas_righthand = [0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018, 0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031] sigmas_wholebody = sigmas_body + sigmas_foot + sigmas_face + sigmas_lefthand + sigmas_righthand # if silence: # nullwrite = NullWriter() # oldstdout = sys.stdout # sys.stdout = nullwrite # disable output # cocoGt = COCO(ann_file) # cocoDt = cocoGt.loadRes(res_file) # res = {} # #cocoEval = COCOeval(cocoGt, cocoDt, ann_type) # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_body', np.array(sigmas_body+sigmas_foot),use_area=True) # cocoEval.evaluate() # cocoEval.accumulate() # cocoEval.summarize() # res['body'] = cocoEval.stats # # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_foot', np.array(sigmas_foot), use_area=True) # # cocoEval.evaluate() # # cocoEval.accumulate() # # cocoEval.summarize() # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_face', np.array(sigmas_face), use_area=True) # cocoEval.evaluate() # cocoEval.accumulate() # cocoEval.summarize() # res['face'] = cocoEval.stats # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_lefthand', np.array(sigmas_lefthand), use_area=True) # cocoEval.evaluate() # cocoEval.accumulate() # cocoEval.summarize() # res['lefthand'] = cocoEval.stats # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_righthand', np.array(sigmas_righthand), use_area=True) # cocoEval.evaluate() # cocoEval.accumulate() # cocoEval.summarize() # res['righthand'] = cocoEval.stats # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_hand', np.array(sigmas_lefthand+sigmas_righthand), use_area=True) # cocoEval.evaluate() # cocoEval.accumulate() # cocoEval.summarize() # res['hand'] = cocoEval.stats # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_wholebody', np.array(sigmas_wholebody), use_area=True) # cocoEval.evaluate() # cocoEval.accumulate() # cocoEval.summarize() # res['wholebody'] = cocoEval.stats # if silence: # sys.stdout = oldstdout # enable output # #print('cocoeval:',cocoEval.stats[0]) # #if isinstance(cocoEval.stats[0], dict): # # stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', # # 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)'] # parts = ['body', 'face', 'hand', 'lefthand','righthand','wholebody'] # #parts = ['keypoints_body','keypoints_foot','keypoints_face','keypoints_lefthand','keypoints_righthand','keypoints_wholebody'] # info = {} # for i, part in enumerate(parts): # info[part] = res[part][0] # return info if silence: nullwrite = NullWriter() oldstdout = sys.stdout sys.stdout = nullwrite # disable output cocoGt = COCO(ann_file) cocoDt = cocoGt.loadRes(res_file) cocoEval = COCOeval(cocoGt, cocoDt, ann_type) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize() if silence: sys.stdout = oldstdout # enable output if isinstance(cocoEval.stats[0], dict): stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)'] parts = ['body', 'face', 'hand', 'fullbody'] info = {} for i, part in enumerate(parts): info[part] = cocoEval.stats[i][part][0] return info else: stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)'] info_str = {} for ind, name in enumerate(stats_names): info_str[name] = cocoEval.stats[ind] return info_str['AP']
def _do_python_keypoint_eval(self, res_file, res_folder): """Keypoint evaluation using COCOAPI.""" coco_dt = self.coco.loadRes(res_file) coco_eval = COCOeval(self.coco, coco_dt, 'keypoints_body', np.array(self.sigmas_body), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_dt, 'keypoints_foot', np.array(self.sigmas_foot), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_dt, 'keypoints_face', np.array(self.sigmas_face), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_dt, 'keypoints_lefthand', np.array(self.sigmas_lefthand), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_dt, 'keypoints_righthand', np.array(self.sigmas_righthand), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_eval = COCOeval(self.coco, coco_dt, 'keypoints_wholebody', np.array(self.sigmas_wholebody), use_area=True) coco_eval.params.useSegm = None coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats_names = [ 'AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)' ] info_str = [] for ind, name in enumerate(stats_names): info_str.append((name, coco_eval.stats[ind])) return info_str
from xtcocotools.coco import COCO from xtcocotools.cocoeval import COCOeval import numpy as np gt_file = '../annotations/example_crowdpose_val.json' preds = '../annotations/example_crowdpose_preds.json' sigmas = np.array([ .79, .79, .72, .72, .62, .62, 1.07, 1.07, .87, .87, .89, .89, .79, .79 ]) / 10.0 cocoGt = COCO(gt_file) cocoDt = cocoGt.loadRes(preds) cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_crowd', sigmas, use_area=False) cocoEval.evaluate() cocoEval.accumulate() cocoEval.summarize()