class EvalCoco(object):
def __init__(self,
coco,
processor,
annotations_inverse,
*,
max_per_image=20,
small_threshold=0.0):
self.coco = coco
self.processor = processor
self.annotations_inverse = annotations_inverse
self.max_per_image = max_per_image
self.small_threshold = small_threshold
self.predictions = []
self.image_ids = []
self.eval = None
self.decoder_time = 0.0
def stats(self, predictions=None, image_ids=None):
# from pycocotools.cocoeval import COCOeval
if predictions is None:
predictions = self.predictions
if image_ids is None:
image_ids = self.image_ids
cat_ids = self.coco.getCatIds(catNms=['person'])
print('cat_ids', cat_ids)
coco_eval = self.coco.loadRes(predictions)
self.eval = COCOeval(self.coco, coco_eval, iouType='keypoints')
self.eval.params.catIds = cat_ids
if image_ids is not None:
print('image ids', image_ids)
self.eval.params.imgIds = image_ids
self.eval.evaluate()
self.eval.accumulate()
self.eval.summarize()
return self.eval.stats
@staticmethod
def view_keypoints(image_cpu, annotations, gt):
highlight = [5, 7, 9, 11, 13, 15]
keypoint_painter = show.KeypointPainter(highlight=highlight)
skeleton_painter = show.KeypointPainter(show_box=False,
color_connections=True,
markersize=1,
linewidth=6)
with show.canvas() as ax:
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
keypoint_painter.annotations(
ax, [ann for ann in annotations if ann.score() > 0.01])
with show.canvas() as ax:
ax.set_axis_off()
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
skeleton_painter.annotations(
ax, [ann for ann in annotations if ann.score() > 0.01])
instances_gt = None
if gt:
instances_gt = np.stack([a['keypoints'] for a in gt])
# for test: overwrite prediction with true values
# instances = instances_gt.copy()[:1]
with show.canvas() as ax:
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
keypoint_painter.keypoints(ax,
instances_gt,
skeleton=COCO_PERSON_SKELETON)
with show.canvas() as ax:
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
show.white_screen(ax)
keypoint_painter.keypoints(ax,
instances_gt,
color='lightgrey',
skeleton=COCO_PERSON_SKELETON)
keypoint_painter.annotations(
ax, [ann for ann in annotations if ann.score() > 0.01])
def from_predictions(self,
predictions,
meta,
debug=False,
gt=None,
image_cpu=None,
verbose=False,
category_id=1):
image_id = int(meta['image_id'])
self.image_ids.append(image_id)
if debug:
self.view_keypoints(image_cpu, predictions, gt)
predictions = self.annotations_inverse(predictions, meta)
if self.small_threshold:
predictions = [
pred for pred in predictions
if pred.scale(v_th=0.01) >= self.small_threshold
]
if len(predictions) > self.max_per_image:
predictions = predictions[:self.max_per_image]
image_annotations = []
for pred in predictions:
# avoid visible keypoints becoming invisible due to rounding
v_mask = pred.data[:, 2] > 0.0
pred.data[v_mask, 2] = np.maximum(0.01, pred.data[v_mask, 2])
keypoints = np.around(pred.data, 2)
keypoints[:, 2] = 2.0
image_annotations.append({
'image_id':
image_id,
'category_id':
category_id,
'keypoints':
keypoints.reshape(-1).tolist(),
'score':
max(0.01, pred.score()),
})
# force at least one annotation per image (for pycocotools)
if not image_annotations:
image_annotations.append({
'image_id': image_id,
'category_id': category_id,
'keypoints': np.zeros((17 * 3, )).tolist(),
'score': 0.01,
})
if debug:
self.stats(image_annotations, [image_id])
if verbose:
print('detected', image_annotations, len(image_annotations))
oks = self.eval.computeOks(image_id, category_id)
oks[oks < 0.5] = 0.0
print('oks', oks)
print(
'evaluate',
self.eval.evaluateImg(image_id, category_id, (0, 1e5**2),
20))
print(meta)
self.predictions += image_annotations
def write_predictions(self, filename):
predictions = [{
k: v
for k, v in annotation.items()
if k in ('image_id', 'category_id', 'keypoints', 'score')
} for annotation in self.predictions]
with open(filename + '.pred.json', 'w') as f:
json.dump(predictions, f)
print('wrote {}'.format(filename + '.pred.json'))
with zipfile.ZipFile(filename + '.zip', 'w') as myzip:
myzip.write(filename + '.pred.json', arcname='predictions.json')
print('wrote {}'.format(filename + '.zip'))
class EvalCoco(object):
def __init__(self, coco, processor, keypoint_sets_inverse, skeleton=None):
self.coco = coco
self.processor = processor
self.keypoint_sets_inverse = keypoint_sets_inverse
self.skeleton = skeleton or COCO_PERSON_SKELETON
self.predictions = []
self.image_ids = []
self.eval = None
self.decoder_time = 0.0
def stats(self, predictions=None, image_ids=None):
# from pycocotools.cocoeval import COCOeval
if predictions is None:
predictions = self.predictions
if image_ids is None:
image_ids = self.image_ids
cat_ids = self.coco.getCatIds(catNms=['person'])
print('cat_ids', cat_ids)
coco_eval = self.coco.loadRes(predictions)
self.eval = COCOeval(self.coco, coco_eval, iouType='keypoints')
self.eval.params.catIds = cat_ids
if image_ids is not None:
print('image ids', image_ids)
self.eval.params.imgIds = image_ids
self.eval.evaluate()
self.eval.accumulate()
self.eval.summarize()
return self.eval.stats
def view_keypoints(self, image_cpu, annotations, gt):
highlight = [5, 7, 9, 11, 13, 15]
keypoint_painter = show.KeypointPainter(skeleton=self.skeleton, highlight=highlight)
skeleton_painter = show.KeypointPainter(skeleton=self.skeleton,
show_box=False, color_connections=True,
markersize=1, linewidth=6)
with show.canvas() as ax:
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
keypoint_painter.annotations(ax, annotations)
with show.canvas() as ax:
ax.set_axis_off()
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
skeleton_painter.annotations(ax, [ann for ann in annotations if ann.score() > 0.1])
instances_gt = None
if gt:
instances_gt = np.stack([a['keypoints'] for a in gt])
# for test: overwrite prediction with true values
# instances = instances_gt.copy()[:1]
with show.canvas() as ax:
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
keypoint_painter.keypoints(ax, instances_gt)
with show.canvas() as ax:
ax.imshow((np.moveaxis(image_cpu.numpy(), 0, -1) + 2.0) / 4.0)
show.white_screen(ax)
keypoint_painter.keypoints(ax, instances_gt, color='lightgrey')
keypoint_painter.annotations(ax, [ann for ann in annotations if ann.score() > 0.01])
def from_fields(self, fields, meta,
debug=False, gt=None, image_cpu=None, verbose=False,
category_id=1):
if image_cpu is not None:
self.processor.set_cpu_image(None, image_cpu)
start = time.time()
annotations = self.processor.annotations(fields, meta)[:20]
self.decoder_time += time.time() - start
if isinstance(meta, (list, tuple)):
meta = meta[0]
image_id = int(meta['image_id'])
self.image_ids.append(image_id)
if debug:
self.view_keypoints(image_cpu, annotations, gt)
instances, scores = self.processor.keypoint_sets_from_annotations(annotations)
instances = self.keypoint_sets_inverse(instances, meta)
image_annotations = []
for instance, score in zip(instances, scores):
keypoints = np.around(instance, 2)
keypoints[:, 2] = 2.0
image_annotations.append({
'image_id': image_id,
'category_id': category_id,
'keypoints': keypoints.reshape(-1).tolist(),
'score': score,
})
# force at least one annotation per image (for pycocotools)
if not image_annotations:
image_annotations.append({
'image_id': image_id,
'category_id': category_id,
'keypoints': np.zeros((17*3,)).tolist(),
'score': 0.0,
})
if debug:
self.stats(image_annotations, [image_id])
if verbose:
print('detected', image_annotations, len(image_annotations))
oks = self.eval.computeOks(image_id, category_id)
oks[oks < 0.5] = 0.0
print('oks', oks)
print('evaluate', self.eval.evaluateImg(image_id, category_id, (0, 1e5 ** 2), 20))
print(meta)
self.predictions += image_annotations
def write_predictions(self, filename):
predictions = [
{k: v for k, v in annotation.items()
if k in ('image_id', 'category_id', 'keypoints', 'score')}
for annotation in self.predictions
]
with open(filename + '.json', 'w') as f:
json.dump(predictions, f)
print('wrote {}'.format(filename + '.json'))
with zipfile.ZipFile(filename + '.zip', 'w') as myzip:
myzip.write(filename + '.json', arcname='predictions.json')
print('wrote {}'.format(filename + '.zip'))
class MapEvaluator:
def __init__(self, gt_json, cat_ids):
# Eval options
self.iou_threshs = [.5]
self.max_dets = [1, 10, 100]
self.cat_ids = cat_ids
self.coco_gt = COCO(gt_json)
def _init_coco_eval(self, dt_json_data):
coco_dt = self.coco_gt.loadRes(dt_json_data)
self.coco_eval = COCOeval(self.coco_gt, coco_dt, 'bbox')
self.coco_eval.params.iouThrs = np.array(self.iou_threshs)
self.coco_eval.params.maxDets = self.max_dets
self.coco_eval.params.catIds = self.cat_ids
def do_eval(self, dt_json_data):
self._init_coco_eval(dt_json_data)
self.coco_eval.evaluate()
self.coco_eval.accumulate()
def do_eval_and_print(self, dt_json_data):
self._init_coco_eval(dt_json_data)
self.coco_eval.evaluate()
self.coco_eval.accumulate()
self.coco_eval.summarize()
def get_avg_precision_recall(self, t=0, a=0, m=-1):
p = self.coco_eval.eval['precision'][t, :, :, a, m]
r = self.coco_eval.eval['recall'][t, :, a, m]
# Removing -1 entries for classes that have no GT objects.
p = p[:, np.where(r > -1)[0]]
r = r[r > -1]
assert (np.all(p >= 0.) and np.all(r >= 0.))
ap = p.mean(axis=1)
ar = r.mean()
return ap, ar
def evaluateImg(self, imgId, catId, aRng=(-np.inf, np.inf), maxDet=100):
p = self.coco_eval.params
# add backward compatibility if useSegm is specified in params
p.iouType = 'bbox'
p.imgIds = list(np.unique(p.imgIds))
if p.useCats:
p.catIds = list(np.unique(p.catIds))
p.maxDets = sorted(p.maxDets)
self.coco_eval.params = p
self.coco_eval._prepare()
self.coco_eval.ious = {
(imgId, catId): self.coco_eval.computeIoU(imgId, catId)
}
img_eval = self.coco_eval.evaluateImg(imgId, catId, aRng, maxDet)
inds = img_eval['dtIds']
matches = img_eval['dtMatches']
return inds, matches