def _do_python_keypoint_eval(self, res_file):
"""Keypoint evaluation using COCOAPI."""
stats_names = [
'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
with open(res_file, 'r') as file:
res_json = json.load(file)
if not res_json:
info_str = list(zip(stats_names, [
0,
] * len(stats_names)))
return info_str
coco_det = self.coco.loadRes(res_file)
coco_eval = COCOeval(self.coco, coco_det, 'keypoints', self.sigmas)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
info_str = list(zip(stats_names, coco_eval.stats))
return info_str
def _do_python_keypoint_eval(self, res_file):
"""Keypoint evaluation using COCOAPI."""
coco_det = self.coco.loadRes(res_file)
coco_eval = COCOeval(self.coco, coco_det, 'keypoints', self.sigmas)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_names = [
'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
info_str = list(zip(stats_names, coco_eval.stats))
return info_str
def _do_python_keypoint_eval(self, res_file):
"""Keypoint evaluation using COCOAPI."""
coco_dt = self.coco.loadRes(res_file)
coco_eval = COCOeval(self.coco, coco_dt, 'keypoints')
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_names = [
'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
info_str = []
for ind, name in enumerate(stats_names):
info_str.append((name, coco_eval.stats[ind]))
return info_str
0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031
]
sigmas_wholebody = sigmas_body + sigmas_foot + sigmas_face + sigmas_lefthand + sigmas_righthand
cocoGt = COCO(gt_file)
cocoDt = cocoGt.loadRes(preds)
cocoEval = COCOeval(cocoGt,
cocoDt,
'keypoints_body',
np.array(sigmas_body),
use_area=True)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
cocoEval = COCOeval(cocoGt,
cocoDt,
'keypoints_foot',
np.array(sigmas_foot),
use_area=True)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
cocoEval = COCOeval(cocoGt,
cocoDt,
'keypoints_face',
np.array(sigmas_face),
use_area=True)
def _do_python_keypoint_eval(self, res_file):
"""Keypoint evaluation using COCOAPI."""
coco_det = self.coco.loadRes(res_file)
coco_eval = COCOeval(self.coco,
coco_det,
'keypoints_body',
np.array(self.sigmas_body),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_det,
'keypoints_foot',
np.array(self.sigmas_foot),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_det,
'keypoints_face',
np.array(self.sigmas_face),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_det,
'keypoints_lefthand',
np.array(self.sigmas_lefthand),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_det,
'keypoints_righthand',
np.array(self.sigmas_righthand),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_det,
'keypoints_wholebody',
np.array(self.sigmas_wholebody),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_names = [
'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
info_str = list(zip(stats_names, coco_eval.stats))
return info_str
def _do_python_keypoint_eval(self, res_file):
"""Keypoint evaluation using COCOAPI."""
coco_det = self.coco.loadRes(res_file)
cuts = np.cumsum([
0, self.body_num, self.foot_num, self.face_num, self.left_hand_num,
self.right_hand_num
])
coco_eval = COCOeval(
self.coco,
coco_det,
'keypoints_body',
self.sigmas[cuts[0]:cuts[1]],
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(
self.coco,
coco_det,
'keypoints_foot',
self.sigmas[cuts[1]:cuts[2]],
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(
self.coco,
coco_det,
'keypoints_face',
self.sigmas[cuts[2]:cuts[3]],
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(
self.coco,
coco_det,
'keypoints_lefthand',
self.sigmas[cuts[3]:cuts[4]],
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(
self.coco,
coco_det,
'keypoints_righthand',
self.sigmas[cuts[4]:cuts[5]],
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(
self.coco,
coco_det,
'keypoints_wholebody',
self.sigmas,
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_names = [
'AP', 'AP .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
info_str = list(zip(stats_names, coco_eval.stats))
return info_str
def cocoapi_eval(jsonfile,
style,
coco_gt=None,
anno_file=None,
max_dets=(100, 300, 1000),
classwise=False,
sigmas=None,
use_area=True):
"""
Args:
jsonfile (str): Evaluation json file, eg: bbox.json, mask.json.
style (str): COCOeval style, can be `bbox` , `segm` , `proposal`, `keypoints` and `keypoints_crowd`.
coco_gt (str): Whether to load COCOAPI through anno_file,
eg: coco_gt = COCO(anno_file)
anno_file (str): COCO annotations file.
max_dets (tuple): COCO evaluation maxDets.
classwise (bool): Whether per-category AP and draw P-R Curve or not.
sigmas (nparray): keypoint labelling sigmas.
use_area (bool): If gt annotations (eg. CrowdPose, AIC)
do not have 'area', please set use_area=False.
"""
assert coco_gt != None or anno_file != None
if style == 'keypoints_crowd':
#please install xtcocotools==1.6
from xtcocotools.coco import COCO
from xtcocotools.cocoeval import COCOeval
else:
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
if coco_gt == None:
coco_gt = COCO(anno_file)
logger.info("Start evaluate...")
coco_dt = coco_gt.loadRes(jsonfile)
if style == 'proposal':
coco_eval = COCOeval(coco_gt, coco_dt, 'bbox')
coco_eval.params.useCats = 0
coco_eval.params.maxDets = list(max_dets)
elif style == 'keypoints_crowd':
coco_eval = COCOeval(coco_gt, coco_dt, style, sigmas, use_area)
else:
coco_eval = COCOeval(coco_gt, coco_dt, style)
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
if classwise:
# Compute per-category AP and PR curve
try:
from terminaltables import AsciiTable
except Exception as e:
logger.error(
'terminaltables not found, plaese install terminaltables. '
'for example: `pip install terminaltables`.')
raise e
precisions = coco_eval.eval['precision']
cat_ids = coco_gt.getCatIds()
# precision: (iou, recall, cls, area range, max dets)
assert len(cat_ids) == precisions.shape[2]
results_per_category = []
for idx, catId in enumerate(cat_ids):
# area range index 0: all area ranges
# max dets index -1: typically 100 per image
nm = coco_gt.loadCats(catId)[0]
precision = precisions[:, :, idx, 0, -1]
precision = precision[precision > -1]
if precision.size:
ap = np.mean(precision)
else:
ap = float('nan')
results_per_category.append(
(str(nm["name"]), '{:0.3f}'.format(float(ap))))
pr_array = precisions[0, :, idx, 0, 2]
recall_array = np.arange(0.0, 1.01, 0.01)
draw_pr_curve(pr_array,
recall_array,
out_dir=style + '_pr_curve',
file_name='{}_precision_recall_curve.jpg'.format(
nm["name"]))
num_columns = min(6, len(results_per_category) * 2)
results_flatten = list(itertools.chain(*results_per_category))
headers = ['category', 'AP'] * (num_columns // 2)
results_2d = itertools.zip_longest(
*[results_flatten[i::num_columns] for i in range(num_columns)])
table_data = [headers]
table_data += [result for result in results_2d]
table = AsciiTable(table_data)
logger.info('Per-category of {} AP: \n{}'.format(style, table.table))
logger.info("per-category PR curve has output to {} folder.".format(
style + '_pr_curve'))
# flush coco evaluation result
sys.stdout.flush()
return coco_eval.stats
def _stats(self, predictions=None, image_ids=None):
# from pycocotools.cocoeval import COCOeval
if predictions is None:
predictions = self.predictions
if image_ids is None:
image_ids = self.image_ids
coco_eval = self.coco.loadRes(predictions)
for count in coco_eval.anns:
ann_orig = copy.deepcopy(coco_eval.anns[count])
coco_eval.anns[count]["lefthand_kpts"] = ann_orig["keypoints"][91 * 3:112 * 3]
coco_eval.anns[count]["righthand_kpts"] = ann_orig["keypoints"][112 * 3:133 * 3]
coco_eval.anns[count]["face_kpts"] = ann_orig["keypoints"][23 * 3:91 * 3]
coco_eval.anns[count]["foot_kpts"] = ann_orig["keypoints"][17 * 3:23 * 3]
coco_eval.anns[count]["keypoints"] = ann_orig["keypoints"][0:17 * 3]
coco_eval.anno_file = coco_eval.anns # xtpycocotools style
self.coco.anno_file = self.coco.anns
eval_body = COCOeval(self.coco, coco_eval, 'keypoints_body', np.array(self.sigmas_body),
use_area=True)
eval_body.evaluate()
eval_body.accumulate()
eval_body.summarize()
eval_foot = COCOeval(self.coco, coco_eval, 'keypoints_foot', np.array(self.sigmas_foot),
use_area=True)
eval_foot.evaluate()
eval_foot.accumulate()
eval_foot.summarize()
eval_face = COCOeval(self.coco, coco_eval, 'keypoints_face', np.array(self.sigmas_face),
use_area=True)
eval_face.evaluate()
eval_face.accumulate()
eval_face.summarize()
eval_lh = COCOeval(self.coco, coco_eval, 'keypoints_lefthand',
np.array(self.sigmas_lefthand), use_area=True)
eval_lh.evaluate()
eval_lh.accumulate()
eval_lh.summarize()
eval_rh = COCOeval(self.coco, coco_eval, 'keypoints_righthand',
np.array(self.sigmas_righthand), use_area=True)
eval_rh.evaluate()
eval_rh.accumulate()
eval_rh.summarize()
eval_wb = COCOeval(self.coco, coco_eval, 'keypoints_wholebody',
np.array(self.sigmas_wholebody), use_area=True)
eval_wb.evaluate()
eval_wb.accumulate()
eval_wb.summarize()
return eval_wb.stats
def evaluate_mAP(res_file, ann_type='bbox', ann_file='./data/coco/annotations/person_keypoints_val2017.json', silence=True):
"""Evaluate mAP result for coco dataset.
Parameters
----------
res_file: str
Path to result json file.
ann_type: str
annotation type, including: `bbox`, `segm`, `keypoints`.
ann_file: str
Path to groundtruth file.
silence: bool
True: disable running log.
"""
class NullWriter(object):
def write(self, arg):
pass
# ann_file = os.path.join('./data/coco/annotations/', ann_file)
sigmas_body = [0.026, 0.025, 0.025, 0.035, 0.035, 0.079, 0.079, 0.072, 0.072, 0.062, 0.062, 0.107, 0.107, 0.087,0.087, 0.089, 0.089]
sigmas_foot = [0.068, 0.066, 0.066, 0.092, 0.094, 0.094]
sigmas_face = [0.042, 0.043, 0.044, 0.043, 0.040, 0.035, 0.031, 0.025, 0.020, 0.023, 0.029, 0.032, 0.037, 0.038, 0.043,0.041, 0.045, 0.013, 0.012, 0.011, 0.011, 0.012, 0.012, 0.011, 0.011, 0.013, 0.015, 0.009, 0.007, 0.007,0.007, 0.012, 0.009, 0.008, 0.016, 0.010, 0.017, 0.011, 0.009, 0.011, 0.009, 0.007, 0.013, 0.008, 0.011,0.012, 0.010, 0.034, 0.008, 0.008, 0.009, 0.008, 0.008, 0.007, 0.010, 0.008, 0.009, 0.009, 0.009, 0.007, 0.007, 0.008, 0.011, 0.008, 0.008, 0.008, 0.01, 0.008]
sigmas_lefthand = [0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018, 0.024, 0.022, 0.026, 0.017,0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031]
sigmas_righthand = [0.029, 0.022, 0.035, 0.037, 0.047, 0.026, 0.025, 0.024, 0.035, 0.018, 0.024, 0.022, 0.026, 0.017, 0.021, 0.021, 0.032, 0.02, 0.019, 0.022, 0.031]
sigmas_wholebody = sigmas_body + sigmas_foot + sigmas_face + sigmas_lefthand + sigmas_righthand
# if silence:
# nullwrite = NullWriter()
# oldstdout = sys.stdout
# sys.stdout = nullwrite # disable output
# cocoGt = COCO(ann_file)
# cocoDt = cocoGt.loadRes(res_file)
# res = {}
# #cocoEval = COCOeval(cocoGt, cocoDt, ann_type)
# cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_body', np.array(sigmas_body+sigmas_foot),use_area=True)
# cocoEval.evaluate()
# cocoEval.accumulate()
# cocoEval.summarize()
# res['body'] = cocoEval.stats
# # cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_foot', np.array(sigmas_foot), use_area=True)
# # cocoEval.evaluate()
# # cocoEval.accumulate()
# # cocoEval.summarize()
# cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_face', np.array(sigmas_face), use_area=True)
# cocoEval.evaluate()
# cocoEval.accumulate()
# cocoEval.summarize()
# res['face'] = cocoEval.stats
# cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_lefthand', np.array(sigmas_lefthand), use_area=True)
# cocoEval.evaluate()
# cocoEval.accumulate()
# cocoEval.summarize()
# res['lefthand'] = cocoEval.stats
# cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_righthand', np.array(sigmas_righthand), use_area=True)
# cocoEval.evaluate()
# cocoEval.accumulate()
# cocoEval.summarize()
# res['righthand'] = cocoEval.stats
# cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_hand', np.array(sigmas_lefthand+sigmas_righthand), use_area=True)
# cocoEval.evaluate()
# cocoEval.accumulate()
# cocoEval.summarize()
# res['hand'] = cocoEval.stats
# cocoEval = COCOeval(cocoGt, cocoDt, 'keypoints_wholebody', np.array(sigmas_wholebody), use_area=True)
# cocoEval.evaluate()
# cocoEval.accumulate()
# cocoEval.summarize()
# res['wholebody'] = cocoEval.stats
# if silence:
# sys.stdout = oldstdout # enable output
# #print('cocoeval:',cocoEval.stats[0])
# #if isinstance(cocoEval.stats[0], dict):
# # stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)',
# # 'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)']
# parts = ['body', 'face', 'hand', 'lefthand','righthand','wholebody']
# #parts = ['keypoints_body','keypoints_foot','keypoints_face','keypoints_lefthand','keypoints_righthand','keypoints_wholebody']
# info = {}
# for i, part in enumerate(parts):
# info[part] = res[part][0]
# return info
if silence:
nullwrite = NullWriter()
oldstdout = sys.stdout
sys.stdout = nullwrite # disable output
cocoGt = COCO(ann_file)
cocoDt = cocoGt.loadRes(res_file)
cocoEval = COCOeval(cocoGt, cocoDt, ann_type)
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
if silence:
sys.stdout = oldstdout # enable output
if isinstance(cocoEval.stats[0], dict):
stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)',
'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)']
parts = ['body', 'face', 'hand', 'fullbody']
info = {}
for i, part in enumerate(parts):
info[part] = cocoEval.stats[i][part][0]
return info
else:
stats_names = ['AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)',
'AR', 'AR .5', 'AR .75', 'AR (M)', 'AR (L)']
info_str = {}
for ind, name in enumerate(stats_names):
info_str[name] = cocoEval.stats[ind]
return info_str['AP']
def _do_python_keypoint_eval(self, res_file, res_folder):
"""Keypoint evaluation using COCOAPI."""
coco_dt = self.coco.loadRes(res_file)
coco_eval = COCOeval(self.coco,
coco_dt,
'keypoints_body',
np.array(self.sigmas_body),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_dt,
'keypoints_foot',
np.array(self.sigmas_foot),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_dt,
'keypoints_face',
np.array(self.sigmas_face),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_dt,
'keypoints_lefthand',
np.array(self.sigmas_lefthand),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_dt,
'keypoints_righthand',
np.array(self.sigmas_righthand),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
coco_eval = COCOeval(self.coco,
coco_dt,
'keypoints_wholebody',
np.array(self.sigmas_wholebody),
use_area=True)
coco_eval.params.useSegm = None
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_names = [
'AP', 'Ap .5', 'AP .75', 'AP (M)', 'AP (L)', 'AR', 'AR .5',
'AR .75', 'AR (M)', 'AR (L)'
]
info_str = []
for ind, name in enumerate(stats_names):
info_str.append((name, coco_eval.stats[ind]))
return info_str
