def _coco_evaluation(self, predicts, answers): coco_res = [] ann = { 'images': [], 'info': '', 'type': 'captions', 'annotations': [], 'licenses': '' } for i, (predict, answer) in enumerate(zip(predicts, answers)): predict_cap = ' '.join(predict) answer_cap = ' '.join(answer).replace('_UNK', '_UNKNOWN') ann['images'].append({'id': i}) ann['annotations'].append({ 'caption': answer_cap, 'id': i, 'image_id': i }) coco_res.append({'caption': predict_cap, 'id': i, 'image_id': i}) coco = COCO(ann) coco_res = coco.loadRes(coco_res) coco_eval = COCOEvalCap(coco, coco_res) coco_eval.evaluate() return coco_eval.eval
def _coco_evaluation(self, predicts, answers): coco_res = [] ann = { 'images': [], 'info': '', 'type': 'captions', 'annotations': [], 'licenses': '' } for i, (predict, _answers) in enumerate(zip(predicts, answers)): predict_cap = ' '.join(predict) if type(_answers) == str: _answers = [_answers] answer_caps = [] for _answer in _answers: answer_cap = ' '.join(_answer).replace('_UNK', '_UNKNOWN') answer_caps.append(answer_cap) ann['images'].append({'id': i}) for answer_cap in answer_caps: ann['annotations'].append({ 'caption': answer_cap, 'id': i, 'image_id': i }) coco_res.append({'caption': predict_cap, 'id': i, 'image_id': i}) with contextlib.redirect_stdout(None): coco = COCO(ann) coco_res = coco.loadRes(coco_res) coco_eval = COCOEvalCap(coco, coco_res) coco_eval.evaluate() return coco_eval.eval
class COCODetection(data.Dataset): """VOC Detection Dataset Object input is image, target is annotation Arguments: root (string): filepath to VOCdevkit folder. image_set (string): imageset to use (eg. 'train', 'val', 'test') transform (callable, optional): transformation to perform on the input image target_transform (callable, optional): transformation to perform on the target `annotation` (eg: take in caption string, return tensor of word indices) dataset_name (string, optional): which dataset to load (default: 'VOC2007') """ def __init__(self, root, image_sets, preproc=None, target_transform=None, dataset_name='COCO'): self.root = root self.cache_path = os.path.join(self.root, 'cache') self.image_set = image_sets self.preproc = preproc self.target_transform = target_transform self.name = dataset_name self.ids = list() self.annotations = list() for (year, image_set) in image_sets: coco_name = image_set + year image_root = os.path.join(root, 'images', _PREDEFINED_SPLITS_COCO[coco_name][0]) annofile = os.path.join(root, _PREDEFINED_SPLITS_COCO[coco_name][1]) self._COCO = COCO(annofile) self.coco_name = coco_name self.class_name = self._get_coco_instances_meta() self.num_classes = len(self.class_name) self.img_ids = sorted(self._COCO.imgs.keys()) imgs = self._COCO.loadImgs(self.img_ids) self.ids.extend( [os.path.join(image_root, img["file_name"]) for img in imgs]) self.annotations.extend( self._load_coco_annotations(coco_name, self.img_ids, self._COCO)) def _load_coco_annotations(self, coco_name, indexes, _COCO): cache_file = os.path.join(self.cache_path, coco_name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = pickle.load(fid) print('{} gt roidb loaded from {}'.format(coco_name, cache_file)) return roidb gt_roidb = [ self._annotation_from_index(index, _COCO) for index in indexes ] if not os.path.exists(os.path.dirname(cache_file)): os.makedirs(os.path.dirname(cache_file)) with open(cache_file, 'wb') as fid: pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL) print('wrote gt roidb to {}'.format(cache_file)) return gt_roidb def _get_coco_instances_meta(self): thing_ids = self._COCO.getCatIds() cats = self._COCO.loadCats(thing_ids) cats_name = [c['name'] for c in cats] self._class_to_coco_cat_id = dict(zip(cats_name, thing_ids)) voc_inds = (0, 1, 2, 3, 4, 5, 6, 8, 14, 15, 16, 17, 18, 19, 39, 56, 57, 58, 60, 62) nonvoc_inds = tuple([i for i in range(80) if i not in voc_inds]) if 'nonvoc' in self.coco_name: self.id_map = nonvoc_inds thing_ids = [thing_ids[i] for i in self.id_map] thing_classes = [cats_name[k] for k in self.id_map] elif 'voc' in self.coco_name: self.id_map = voc_inds thing_ids = [thing_ids[i] for i in self.id_map] thing_classes = [cats_name[k] for k in self.id_map] self._thing_dataset_id_to_contiguous_id = { k: i for i, k in enumerate(thing_ids, 1) } return thing_classes def _annotation_from_index(self, index, _COCO): """ Loads COCO bounding-box instance annotations. Crowd instances are handled by marking their overlaps (with all categories) to -1. This overlap value means that crowd "instances" are excluded from training. """ im_ann = _COCO.loadImgs(index)[0] width = im_ann['width'] height = im_ann['height'] annIds = _COCO.getAnnIds(imgIds=index, iscrowd=None) objs = _COCO.loadAnns(annIds) # Sanitize bboxes -- some are invalid valid_objs = [] for obj in objs: x1 = np.max((0, obj['bbox'][0])) y1 = np.max((0, obj['bbox'][1])) x2 = np.min((width - 1, x1 + np.max((0, obj['bbox'][2] - 1)))) y2 = np.min((height - 1, y1 + np.max((0, obj['bbox'][3] - 1)))) if obj['area'] > 0 and x2 >= x1 and y2 >= y1: obj['clean_bbox'] = [x1, y1, x2, y2] valid_objs.append(obj) objs = valid_objs num_objs = len(objs) res = np.zeros((num_objs, 5)) # Lookup table to map from COCO category ids to our internal class # indices for ix, obj in enumerate(objs): cls = self._thing_dataset_id_to_contiguous_id[obj['category_id']] res[ix, 0:4] = obj['clean_bbox'] res[ix, 4] = cls return res def __getitem__(self, index): img_id = self.ids[index] target = self.annotations[index] img = cv2.imread(img_id, cv2.IMREAD_COLOR) height, width, _ = img.shape if self.target_transform is not None: target = self.target_transform(target) if self.preproc is not None: img, target = self.preproc(img, target) # in order to be compatible with mixup weight = np.ones((target.shape[0], 1)) target = np.hstack((target, weight)) return img, target def __len__(self): return len(self.ids) def pull_image(self, index): '''Returns the original image object at index in PIL form Note: not using self.__getitem__(), as any transformations passed in could mess up this functionality. Argument: index (int): index of img to show Return: PIL img ''' img_id = self.ids[index] return cv2.imread(img_id, cv2.IMREAD_COLOR) def pull_tensor(self, index): '''Returns the original image at an index in tensor form Note: not using self.__getitem__(), as any transformations passed in could mess up this functionality. Argument: index (int): index of img to show Return: tensorized version of img, squeezed ''' return torch.Tensor(self.pull_image(index)).unsqueeze_(0) def _do_detection_eval(self, res_file): coco_dt = self._COCO.loadRes(res_file) coco_eval = COCOeval(self._COCO, coco_dt, 'bbox') coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() results = OrderedDict() results['bbox'] = self._derive_coco_results( coco_eval, 'bbox', class_names=self.class_name) print_csv_format(results) def _coco_results_one_category(self, boxes, cat_id): results = [] for im_ind, index in enumerate(self.img_ids): dets = boxes[im_ind].astype(np.float) if dets == []: continue scores = dets[:, -1] xs = dets[:, 0] ys = dets[:, 1] ws = dets[:, 2] - xs + 1 hs = dets[:, 3] - ys + 1 results.extend([{ 'image_id': index, 'category_id': cat_id, 'bbox': [xs[k], ys[k], ws[k], hs[k]], 'score': scores[k] } for k in range(dets.shape[0])]) return results def _write_coco_results_file(self, all_boxes, res_file): # [{"image_id": 42, # "category_id": 18, # "bbox": [258.15,41.29,348.26,243.78], # "score": 0.236}, ...] results = [] for cls_ind, cls in enumerate(self.class_name, 1): print('Collecting {} results ({:d}/{:d})'.format( cls, cls_ind, self.num_classes)) coco_cat_id = self._class_to_coco_cat_id[cls] results.extend( self._coco_results_one_category(all_boxes[cls_ind], coco_cat_id)) print('Writing results json to {}'.format(res_file)) with open(res_file, 'w') as fid: json.dump(results, fid) fid.flush() def evaluate_detections(self, all_boxes, output_dir): res_file = os.path.join(output_dir, 'detections_' + self.coco_name + '_results') res_file += '.json' self._write_coco_results_file(all_boxes, res_file) # Only do evaluation on non-test sets if self.coco_name.find('test') == -1: self._do_detection_eval(res_file) def _derive_coco_results(self, coco_eval, iou_type, class_names=None): """ Derive the desired score numbers from summarized COCOeval. Args: coco_eval (None or COCOEval): None represents no predictions from model. iou_type (str): class_names (None or list[str]): if provided, will use it to predict per-category AP. Returns: a dict of {metric name: score} """ metrics = { "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"], "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"], "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], }[iou_type] if coco_eval is None: print("No predictions from the model! Set scores to -1") return {metric: -1 for metric in metrics} # the standard metrics results = { metric: float(coco_eval.stats[idx] * 100) for idx, metric in enumerate(metrics) } print("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) if class_names is None or len(class_names) <= 1: return results # Compute per-category AP # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa precisions = coco_eval.eval["precision"] # precision has dims (iou, recall, cls, area range, max dets) # assert len(class_names) == precisions.shape[2] results_per_category = [] for idx, name in zip(self.id_map, class_names): # area range index 0: all area ranges # max dets index -1: typically 100 per image precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] ap = np.mean(precision) if precision.size else float("nan") results_per_category.append(("{}".format(name), float(ap * 100))) # tabulate it N_COLS = min(6, len(results_per_category) * 2) results_flatten = list(itertools.chain(*results_per_category)) results_2d = itertools.zip_longest( *[results_flatten[i::N_COLS] for i in range(N_COLS)]) table = tabulate( results_2d, tablefmt="pipe", floatfmt=".3f", headers=["category", "AP"] * (N_COLS // 2), numalign="left", ) print("Per-category {} AP: \n".format(iou_type) + table) results.update({"AP-" + name: ap for name, ap in results_per_category}) return results
def evaluate_detections(FPS=None): print('Running demo for *%s* results.' % (annType)) # initialize COCO ground truth api annFile = '%s/annotations/%s_%s.json' % (dataDir, prefix, dataType) print(annFile) cocoGt = COCO(annFile) # initialize COCO detections api cocoDt = cocoGt.loadRes(resFile) imgIds = cocoGt.getImgIds() # imgIds = imgIds[0:100] # imgId = imgIds[np.random.randint(100)] # running evaluation cocoEval = COCOeval(cocoGt, cocoDt, annType) cocoEval.params.imgIds = imgIds cocoEval.evaluate() cocoEval.accumulate() means = cocoEval.summarize() with open(os.path.join(output_dir, str(int(means[0] * 10000)) + '.txt'), 'w') as res_file: res_file.write('CUDA: ' + str(args.cuda) + '\n') res_file.write('model_dir: ' + args.model_dir + '\n') res_file.write('iteration: ' + args.iteration + '\n') res_file.write('model_name: ' + args.model_name + '\n') res_file.write('backbone : ' + args.backbone + '\n') if args.backbone in ['RefineDet_VGG']: res_file.write('refine : ' + str(args.refine) + '\n') res_file.write('deform : ' + str(args.deform) + '\n') res_file.write('multi-head : ' + str(args.multihead) + '\n') res_file.write('ssd_dim: ' + str(args.ssd_dim) + '\n') res_file.write('confidence_threshold: ' + str(args.confidence_threshold) + '\n') res_file.write('nms_threshold: ' + str(args.nms_threshold) + '\n') res_file.write('top_k: ' + str(args.top_k) + '\n') res_file.write('dataset_name: ' + str(args.dataset_name) + '\n') res_file.write('set_file_name: ' + str(args.set_file_name) + '\n') res_file.write('detection: ' + str(args.detection) + '\n') res_file.write('~~~~~~~~~~~~~~~~~\n') res_file.write( 'Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = {:.4f}\n' .format(means[0])) res_file.write( 'Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = {:.4f}\n' .format(means[1])) res_file.write( 'Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = {:.4f}\n' .format(means[2])) res_file.write( 'Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = {:.4f}\n' .format(means[3])) res_file.write( 'Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = {:.4f}\n' .format(means[4])) res_file.write( 'Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = {:.4f}\n' .format(means[5])) res_file.write( 'Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 1 ] = {:.4f}\n' .format(means[6])) res_file.write( 'Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets= 10 ] = {:.4f}\n' .format(means[7])) res_file.write( 'Average Recall (AR) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = {:.4f}\n' .format(means[8])) res_file.write( 'Average Recall (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = {:.4f}\n' .format(means[8])) res_file.write( 'Average Recall (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = {:.4f}\n' .format(means[10])) res_file.write( 'Average Recall (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = {:.4f}\n' .format(means[11])) if FPS: for i, f in enumerate(FPS): res_file.write(str(i) + ': FPS = {:.4f}\n'.format(f))