def compute_map(labels_and_predictions, coco_gt, use_cpp_extension=True): """Use model predictions to compute mAP. The evaluation code is largely copied from the MLPerf reference implementation. While it is possible to write the evaluation as a tensor metric and use Estimator.evaluate(), this approach was selected for simplicity and ease of duck testing. Args: labels_and_predictions: A map from TPU predict method. coco_gt: ground truch COCO object. use_cpp_extension: use cocoeval C++ library. Returns: Evaluation result. """ predictions = [] tic = time.time() for example in labels_and_predictions: if constants.IS_PADDED in example and example[constants.IS_PADDED]: continue htot, wtot, _ = example[constants.RAW_SHAPE] pred_box = example['pred_box'] pred_scores = example['pred_scores'] indices = example['indices'] loc, label, prob = decode_single(pred_box, pred_scores, indices, constants.OVERLAP_CRITERIA, constants.MAX_NUM_EVAL_BOXES, constants.MAX_NUM_EVAL_BOXES) for loc_, label_, prob_ in zip(loc, label, prob): # Ordering convention differs, hence [1], [0] rather than [0], [1] predictions.append([ int(example[constants.SOURCE_ID]), loc_[1] * wtot, loc_[0] * htot, (loc_[3] - loc_[1]) * wtot, (loc_[2] - loc_[0]) * htot, prob_, constants.CLASS_INV_MAP[label_] ]) toc = time.time() tf.logging.info('Prepare predictions DONE (t={:0.2f}s).'.format(toc - tic)) if use_cpp_extension: coco_dt = coco_gt.LoadRes(np.array(predictions, dtype=np.float32)) coco_eval = COCOeval(coco_gt, coco_dt, iou_type='bbox') coco_eval.Evaluate() coco_eval.Accumulate() coco_eval.Summarize() stats = coco_eval.GetStats() else: coco_dt = coco_gt.loadRes(np.array(predictions)) coco_eval = COCOeval(coco_gt, coco_dt, iouType='bbox') coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() stats = coco_eval.stats print('Current AP: {:.5f}'.format(stats[0])) metric_names = [ 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'ARmax1', 'ARmax10', 'ARmax100', 'ARs', 'ARm', 'ARl' ] coco_time = time.time() tf.logging.info('COCO eval DONE (t={:0.2f}s).'.format(coco_time - toc)) # Prefix with "COCO" to group in TensorBoard. return {'COCO/' + key: value for key, value in zip(metric_names, stats)}