def evaluate(self): """Evaluates with detections from all images with COCO API. Returns: coco_metric: float numpy array with shape [24] representing the coco-style evaluation metrics (box and mask). """ if not self._annotation_file: gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset( self._groundtruths) coco_gt = coco_utils.COCOWrapper( eval_type=('mask' if self._include_mask else 'box'), gt_dataset=gt_dataset) else: coco_gt = self._coco_gt coco_predictions = coco_utils.convert_predictions_to_coco_annotations( self._predictions) coco_dt = coco_gt.loadRes(predictions=coco_predictions) image_ids = [ann['image_id'] for ann in coco_predictions] coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_metrics = coco_eval.stats if self._include_mask: mcoco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='segm') mcoco_eval.params.imgIds = image_ids mcoco_eval.evaluate() mcoco_eval.accumulate() mcoco_eval.summarize() if self._mask_eval_class == 'all': metrics = np.hstack((coco_metrics, mcoco_eval.stats)) else: mask_coco_metrics = mcoco_eval.category_stats val_catg_idx = np.isin(mcoco_eval.params.catIds, self._eval_categories) # Gather the valid evaluation of the eval categories. if np.any(val_catg_idx): mean_val_metrics = [] for mid in range(len(self._metric_names) // 2): mean_val_metrics.append( np.nanmean(mask_coco_metrics[mid][val_catg_idx])) mean_val_metrics = np.array(mean_val_metrics) else: mean_val_metrics = np.zeros(len(self._metric_names) // 2) metrics = np.hstack((coco_metrics, mean_val_metrics)) else: metrics = coco_metrics # Cleans up the internal variables in order for a fresh eval next time. self.reset() metrics_dict = {} for i, name in enumerate(self._metric_names): metrics_dict[name] = metrics[i].astype(np.float32) return metrics_dict
def __init__(self, annotation_file, include_mask, need_rescale_bboxes=True, per_category_metrics=False): """Constructs COCO evaluation class. The class provides the interface to metrics_fn in TPUEstimator. The _update_op() takes detections from each image and push them to self.detections. The _evaluate() loads a JSON file in COCO annotation format as the groundtruths and runs COCO evaluation. Args: annotation_file: a JSON file that stores annotations of the eval dataset. If `annotation_file` is None, groundtruth annotations will be loaded from the dataloader. include_mask: a boolean to indicate whether or not to include the mask eval. need_rescale_bboxes: If true bboxes in `predictions` will be rescaled back to absolute values (`image_info` is needed in this case). per_category_metrics: Whether to return per category metrics. """ if annotation_file: if annotation_file.startswith('gs://'): _, local_val_json = tempfile.mkstemp(suffix='.json') tf.gfile.Remove(local_val_json) tf.gfile.Copy(annotation_file, local_val_json) atexit.register(tf.gfile.Remove, local_val_json) else: local_val_json = annotation_file self._coco_gt = coco_utils.COCOWrapper( eval_type=('mask' if include_mask else 'box'), annotation_file=local_val_json) self._annotation_file = annotation_file self._include_mask = include_mask self._per_category_metrics = per_category_metrics self._metric_names = [ 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'ARmax1', 'ARmax10', 'ARmax100', 'ARs', 'ARm', 'ARl' ] self._required_prediction_fields = [ 'source_id', 'num_detections', 'detection_classes', 'detection_scores', 'detection_boxes' ] self._need_rescale_bboxes = need_rescale_bboxes if self._need_rescale_bboxes: self._required_prediction_fields.append('image_info') self._required_groundtruth_fields = [ 'source_id', 'height', 'width', 'classes', 'boxes' ] if self._include_mask: mask_metric_names = ['mask_' + x for x in self._metric_names] self._metric_names.extend(mask_metric_names) self._required_prediction_fields.extend(['detection_masks']) self._required_groundtruth_fields.extend(['masks']) self.reset()
def evaluate(self): """Evaluates with detections from all images with COCO API. Returns: coco_metric: float numpy array with shape [24] representing the coco-style evaluation metrics (box and mask). """ if not self._annotation_file: logging.info('There is no annotation_file in COCOEvaluator.') gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset( self._groundtruths) coco_gt = coco_utils.COCOWrapper( eval_type=('mask' if self._include_mask else 'box'), gt_dataset=gt_dataset) else: logging.info('Using annotation file: %s', self._annotation_file) coco_gt = self._coco_gt coco_predictions = coco_utils.convert_predictions_to_coco_annotations( self._predictions) coco_dt = coco_gt.loadRes(predictions=coco_predictions) image_ids = [ann['image_id'] for ann in coco_predictions] coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_metrics = coco_eval.stats if self._include_mask: mcoco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='segm') mcoco_eval.params.imgIds = image_ids mcoco_eval.evaluate() mcoco_eval.accumulate() mcoco_eval.summarize() mask_coco_metrics = mcoco_eval.stats if self._include_mask: metrics = np.hstack((coco_metrics, mask_coco_metrics)) else: metrics = coco_metrics # Cleans up the internal variables in order for a fresh eval next time. self.reset() metrics_dict = {} for i, name in enumerate(self._metric_names): metrics_dict[name] = metrics[i].astype(np.float32) return metrics_dict
def evaluate(self): """Evaluates with detections from all images with COCO API. Returns: coco_metric: float numpy array with shape [24] representing the coco-style evaluation metrics (box and mask). """ if not self._annotation_file: logging.info('There is no annotation_file in COCOEvaluator.') gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset( self._groundtruths) coco_gt = coco_utils.COCOWrapper( eval_type=('mask' if self._include_mask else 'box'), gt_dataset=gt_dataset) else: logging.info('Using annotation file: %s', self._annotation_file) coco_gt = self._coco_gt coco_predictions = coco_utils.convert_predictions_to_coco_annotations( self._predictions) coco_dt = coco_gt.loadRes(predictions=coco_predictions) image_ids = [ann['image_id'] for ann in coco_predictions] coco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='bbox') coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() coco_metrics = coco_eval.stats if self._include_mask: mcoco_eval = cocoeval.COCOeval(coco_gt, coco_dt, iouType='segm') mcoco_eval.params.imgIds = image_ids mcoco_eval.evaluate() mcoco_eval.accumulate() mcoco_eval.summarize() mask_coco_metrics = mcoco_eval.stats if self._include_mask: metrics = np.hstack((coco_metrics, mask_coco_metrics)) else: metrics = coco_metrics # Cleans up the internal variables in order for a fresh eval next time. self.reset() metrics_dict = {} for i, name in enumerate(self._metric_names): metrics_dict[name] = metrics[i].astype(np.float32) # Adds metrics per category. if self._per_category_metrics and hasattr(coco_eval, 'category_stats'): for category_index, category_id in enumerate( coco_eval.params.catIds): metrics_dict['Precision mAP ByCategory/{}'.format( category_id )] = coco_eval.category_stats[0][category_index].astype( np.float32) metrics_dict['Precision mAP ByCategory@50IoU/{}'.format( category_id )] = coco_eval.category_stats[1][category_index].astype( np.float32) metrics_dict['Precision mAP ByCategory@75IoU/{}'.format( category_id )] = coco_eval.category_stats[2][category_index].astype( np.float32) metrics_dict['Precision mAP ByCategory (small) /{}'.format( category_id )] = coco_eval.category_stats[3][category_index].astype( np.float32) metrics_dict['Precision mAP ByCategory (medium) /{}'.format( category_id )] = coco_eval.category_stats[4][category_index].astype( np.float32) metrics_dict['Precision mAP ByCategory (large) /{}'.format( category_id )] = coco_eval.category_stats[5][category_index].astype( np.float32) metrics_dict['Recall AR@1 ByCategory/{}'.format( category_id )] = coco_eval.category_stats[6][category_index].astype( np.float32) metrics_dict['Recall AR@10 ByCategory/{}'.format( category_id )] = coco_eval.category_stats[7][category_index].astype( np.float32) metrics_dict['Recall AR@100 ByCategory/{}'.format( category_id )] = coco_eval.category_stats[8][category_index].astype( np.float32) metrics_dict['Recall AR (small) ByCategory/{}'.format( category_id )] = coco_eval.category_stats[9][category_index].astype( np.float32) metrics_dict['Recall AR (medium) ByCategory/{}'.format( category_id )] = coco_eval.category_stats[10][category_index].astype( np.float32) metrics_dict['Recall AR (large) ByCategory/{}'.format( category_id )] = coco_eval.category_stats[11][category_index].astype( np.float32) return metrics_dict
def __init__( self, annotation_file, include_mask, need_rescale_bboxes=True, per_category_metrics=False, include_attributes=False, use_eval_image_sizes=False, score_threshold=0.05, ): """Constructs COCO evaluation class. The class provides the interface to metrics_fn in TPUEstimator. The _update_op() takes detections from each image and push them to self.detections. The _evaluate() loads a JSON file in COCO annotation format as the groundtruths and runs COCO evaluation. Args: annotation_file: a JSON file that stores annotations of the eval dataset. If `annotation_file` is None, groundtruth annotations will be loaded from the dataloader. include_mask: a boolean to indicate whether or not to include the mask eval. need_rescale_bboxes: If true bboxes in `predictions` will be rescaled back to absolute values (`image_info` is needed in this case). per_category_metrics: Whether to return per category metrics. """ if annotation_file: if annotation_file.startswith("gs://"): _, local_val_json = tempfile.mkstemp(suffix=".json") tf.io.gfile.remove(local_val_json) tf.io.gfile.copy(annotation_file, local_val_json) atexit.register(tf.io.gfile.remove, local_val_json) else: local_val_json = annotation_file self._coco_gt = coco_utils.COCOWrapper( eval_type=("mask" if include_mask else "box"), annotation_file=local_val_json, ) self._annotation_file = annotation_file self._include_mask = include_mask self._include_attributes = include_attributes self._per_category_metrics = per_category_metrics self._use_eval_image_sizes = use_eval_image_sizes self._score_threshold = score_threshold self._metric_names = [ "AP", "AP50", "AP75", "APs", "APm", "APl", "ARmax1", "ARmax10", "ARmax100", "ARs", "ARm", "ARl", ] self._required_prediction_fields = [ "source_id", "num_detections", "detection_classes", "detection_scores", "detection_boxes", ] self._need_rescale_bboxes = need_rescale_bboxes if self._need_rescale_bboxes: self._required_prediction_fields.append("image_info") self._required_groundtruth_fields = [ "source_id", "height", "width", "classes", "boxes", ] if self._include_mask: self._required_prediction_fields.extend(["detection_masks"]) self._required_groundtruth_fields.extend(["masks"]) if self._include_attributes: self._required_prediction_fields.extend(["detection_attributes"]) self._required_groundtruth_fields.extend(["attributes"]) self.reset()
def evaluate(self): """Evaluates with detections from all images with COCO API. Returns: coco_metric: float numpy array with shape [24] representing the coco-style evaluation metrics (box and mask). """ if not self._annotation_file: logging.info("There is no annotation_file in COCOEvaluator.") gt_dataset = coco_utils.convert_groundtruths_to_coco_dataset( self._groundtruths) coco_gt = coco_utils.COCOWrapper( eval_type=("mask" if self._include_mask else "box"), gt_dataset=gt_dataset, ) else: logging.info("Using annotation file: %s", self._annotation_file) coco_gt = self._coco_gt logging.info("Loading predictions...") eval_image_sizes = {} if self._use_eval_image_sizes: for image in coco_gt.dataset["images"]: eval_image_sizes[image["id"]] = (image["height"], image["width"]) coco_predictions = coco_utils.convert_predictions_to_coco_annotations( self._predictions, eval_image_sizes, score_threshold=self._score_threshold) coco_dt = coco_gt.loadRes(predictions=coco_predictions) image_ids = [ann["image_id"] for ann in coco_predictions] logging.info("Evaluating bboxes...") coco_eval = COCOeval(coco_gt, coco_dt, iouType="bbox") coco_eval.params.imgIds = image_ids coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() if self._per_category_metrics: coco_eval.summarize_per_category() metrics_dict = self._get_metrics_dict(coco_eval, "bbox") metrics_dict["performance/bbox_ap"] = metrics_dict[ "bbox_performance/AP"] if self._include_mask: logging.info("Evaluating masks...") mcoco_eval = COCOeval(coco_gt, coco_dt, iouType="segm") mcoco_eval.params.imgIds = image_ids mcoco_eval.evaluate() mcoco_eval.accumulate() mcoco_eval.summarize() if self._per_category_metrics: mcoco_eval.summarize_per_category() mask_metrics = self._get_metrics_dict(mcoco_eval, "mask") mask_metrics["performance/mask_ap"] = mask_metrics[ "mask_performance/AP"] metrics_dict.update(mask_metrics) if self._include_attributes: logging.info("Evaluating attributes...") attribute_metrics = evaluate_attributes( coco_gt.dataset["annotations"], coco_dt.dataset["annotations"]) metrics_dict.update(attribute_metrics) # Cleans up the internal variables in order for a fresh eval next time. self.reset() return metrics_dict