def __getitem__(self, idx): img, anno = super(ObstacleCOCOSet, self).__getitem__(idx) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) # masks = [obj["segmentation"] for obj in anno] # masks = SegmentationMask(masks, img.size, mode='poly') # target.add_field("masks", masks) # if anno and "keypoints" in anno[0]: # keypoints = [obj["keypoints"] for obj in anno] # keypoints = PersonKeypoints(keypoints, img.size) # target.add_field("keypoints", keypoints) target = target.clip_to_image(remove_empty=True) if self._transforms is not None: img, target = self._transforms(img, target) return img, target, idx
def get_groundtruth(self, index): img_id = self.ids[index] anno = ET.parse(self._annopath % img_id).getroot() anno = self._preprocess_annotation(anno) height, width = anno["im_info"] target = BoxList(anno["boxes"], (width, height), mode="xyxy") target.add_field("labels", anno["labels"]) # target.add_field("difficult", anno["difficult"]) return target
def __getitem__(self, idx): img, anno = super(COCODataset, self).__getitem__(idx) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] #将label文件转成列表 boxes = [obj["bbox"] for obj in anno] #读取图片中物体的真值框数据[x,y,w,h] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes 生成一个张量 n*4 4这个维度表示的是x,y,w,h, n表示的是 一张图片里有多少个真值框 target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") #按照对应的类 初始化一下 classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) #target.extra_fields["labels"] = classes masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size, mode='poly') target.add_field("masks", masks) if anno and "keypoints" in anno[0]: keypoints = [obj["keypoints"] for obj in anno] keypoints = PersonKeypoints(keypoints, img.size) target.add_field("keypoints", keypoints) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, idx
def __getitem__(self, idx): # from sampled idx to original idx # self.sampled_index = [8, 30, 45, ... ] # 15 frames per video idx = self.sampled_index[idx] img, anno = super(COCODataset, self).__getitem__(idx) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size, mode='poly') target.add_field("masks", masks) if anno and "keypoints" in anno[0]: keypoints = [obj["keypoints"] for obj in anno] keypoints = PersonKeypoints(keypoints, img.size) target.add_field("keypoints", keypoints) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, idx
def im_detect_bbox_aug_vote(arguements, model, images, device): # Collect detections computed under different transformations boxlists_ts = [] for _ in range(len(images)): boxlists_ts.append([]) def add_preds_t(boxlists_t): for i, boxlist_t in enumerate(boxlists_t): if len(boxlists_ts[i]) == 0: # The first one is identity transform, no need to resize the boxlist boxlists_ts[i].append(boxlist_t) else: # Resize the boxlist as the first one boxlists_ts[i].append(boxlist_t.resize(boxlists_ts[i][0].size)) # Compute detections for the original image (identity transform) boxlists_i = im_detect_bbox(arguements, model, images, cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MAX_SIZE_TEST, device) add_preds_t(boxlists_i) # Perform detection on the horizontally flipped image if cfg.TEST.BBOX_AUG.H_FLIP: boxlists_hf = im_detect_bbox_hflip(arguements, model, images, cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MAX_SIZE_TEST, device) add_preds_t(boxlists_hf) for idx, scale in enumerate(cfg.TEST.BBOX_AUG.SCALES): max_size = cfg.TEST.BBOX_AUG.MAX_SIZE min_range = cfg.TEST.BBOX_AUG.SCALE_RANGES[idx][0] max_range = cfg.TEST.BBOX_AUG.SCALE_RANGES[idx][1] if scale < 800: max_size = cfg.INPUT.MAX_SIZE_TEST boxlists_scl = im_detect_bbox_scale(arguements, model, images, scale, max_size, device) boxlists_scl = remove_boxes(boxlists_scl, min_range, max_range) add_preds_t(boxlists_scl) if cfg.TEST.BBOX_AUG.SCALE_H_FLIP: boxlists_scl_hf = im_detect_bbox_scale(arguements, model, images, scale, max_size, device, hflip=True) boxlists_scl_hf = remove_boxes(boxlists_scl_hf, min_range, max_range) add_preds_t(boxlists_scl_hf) # Merge boxlists detected by different bbox aug params boxlists = [] for _, boxlist_ts in enumerate(boxlists_ts): bbox = torch.cat([boxlist_t.bbox for boxlist_t in boxlist_ts]) scores = torch.cat( [boxlist_t.get_field('scores') for boxlist_t in boxlist_ts]) labels = torch.cat( [boxlist_t.get_field('labels') for boxlist_t in boxlist_ts]) boxlist = BoxList(bbox, boxlist_ts[0].size, boxlist_ts[0].mode) boxlist.add_field('scores', scores) boxlist.add_field('labels', labels) boxlists.append(boxlist) results = merge_result_from_multi_scales(boxlists, cfg.TEST.BBOX_AUG.MERGE_TYPE, cfg.TEST.BBOX_AUG.VOTE_TH) return results
def forward_for_single_feature_map(self, anchors, objectness, box_regression): """ Arguments: anchors: list[BoxList] objectness: tensor of size N, A, H, W box_regression: tensor of size N, A * 4, H, W """ device = objectness.device N, A, H, W = objectness.shape # put in the same format as anchors objectness = permute_and_flatten(objectness, N, A, 1, H, W).view(N, -1) objectness = objectness.sigmoid() box_regression = permute_and_flatten(box_regression, N, A, 4, H, W) num_anchors = A * H * W pre_nms_top_n = min(self.pre_nms_top_n, num_anchors) objectness, topk_idx = objectness.topk(pre_nms_top_n, dim=1, sorted=True) batch_idx = torch.arange(N, device=device)[:, None] box_regression = box_regression[batch_idx, topk_idx] image_shapes = [box.size for box in anchors] concat_anchors = torch.cat([a.bbox for a in anchors], dim=0) concat_anchors = concat_anchors.reshape(N, -1, 4)[batch_idx, topk_idx] proposals = self.box_coder.decode(box_regression.view(-1, 4), concat_anchors.view(-1, 4)) proposals = proposals.view(N, -1, 4) result = [] for proposal, score, im_shape in zip(proposals, objectness, image_shapes): boxlist = BoxList(proposal, im_shape, mode="xyxy") boxlist.add_field("objectness", score) boxlist = boxlist.clip_to_image(remove_empty=False) boxlist = remove_small_boxes(boxlist, self.min_size) boxlist = boxlist_nms( boxlist, self.nms_thresh, max_proposals=self.post_nms_top_n, score_field="objectness", ) result.append(boxlist) return result
def get_pos_proposal_indexes(self, locations, box_regression, matched_idxes, targets): locations = torch.cat(locations, dim=0) pos_indexes_for_targets = [] for im in range(len(targets)): pos_indexes_for_targets_per_im = [] box_regression_im = [ box_regression[l][im].detach().view(4, -1).transpose( 0, 1).contiguous() * self.fpn_strides[l] for l in range(len(box_regression)) ] box_regression_im = torch.cat(box_regression_im, dim=0) for t_id in range(len(targets[im])): valid = matched_idxes[im] == t_id if valid.sum() == 0: pos_indexes_for_targets_per_im.append(valid.new_tensor([])) continue valid_location = locations[valid] valid_regression = box_regression_im[valid] detections = torch.stack([ valid_location[:, 0] - valid_regression[:, 0], valid_location[:, 1] - valid_regression[:, 1], valid_location[:, 0] + valid_regression[:, 2], valid_location[:, 1] + valid_regression[:, 3], ], dim=1) detect_boxlist = BoxList(detections, targets[im].size, mode="xyxy") target_boxlist = BoxList(targets[im].bbox[t_id:t_id + 1], targets[im].size, mode="xyxy") match_quality_matrix = boxlist_iou(detect_boxlist, target_boxlist) pos_labels_per_target = torch.zeros_like(valid) iou_in_target = match_quality_matrix[:, 0] if iou_in_target.max() > self.sample_pos_iou_th: pos_in_target = (iou_in_target > self.sample_pos_iou_th) else: pos_in_target = (iou_in_target == iou_in_target.max()) pos_labels_per_target[valid] = pos_in_target pos_indexes_for_targets_per_im.append( pos_labels_per_target.nonzero().squeeze(1)) pos_indexes_for_targets.append(pos_indexes_for_targets_per_im) return pos_indexes_for_targets
def _test_feature_extractors(self, extractors, overwrite_cfgs, overwrite_in_channels): ''' Make sure roi box feature extractors run ''' self.assertGreater(len(extractors), 0) in_channels_default = 64 for name, builder in extractors.items(): print('Testing {}...'.format(name)) if name in overwrite_cfgs: cfg = load_config(overwrite_cfgs[name]) else: # Use default config if config file is not specified cfg = copy.deepcopy(g_cfg) in_channels = overwrite_in_channels.get(name, in_channels_default) fe = builder(cfg, in_channels) self.assertIsNotNone( getattr(fe, 'out_channels', None), 'Need to provide out_channels for feature extractor {}'.format( name)) N, C_in, H, W = 2, in_channels, 24, 32 input = torch.rand([N, C_in, H, W], dtype=torch.float32) bboxes = [[1, 1, 10, 10], [5, 5, 8, 8], [2, 2, 3, 4]] img_size = [384, 512] box_list = BoxList(bboxes, img_size, "xyxy") out = fe([input], [box_list] * N) self.assertEqual(out.shape[:2], torch.Size([N * len(bboxes), fe.out_channels]))
def prepare_boxlist(self, boxes, scores, image_shape): """ Returns BoxList from `boxes` and adds probability scores information as an extra field `boxes` has shape (#detections, 4 * #classes), where each row represents a list of predicted bounding boxes for each of the object classes in the dataset (including the background class). The detections in each row originate from the same object proposal. `scores` has shape (#detection, #classes), where each row represents a list of object detection confidence scores for each of the object classes in the dataset (including the background class). `scores[i, j]`` corresponds to the box at `boxes[i, j * 4:(j + 1) * 4]`. """ boxes = boxes.reshape(-1, 4) scores = scores.reshape(-1) boxlist = BoxList(boxes, image_shape, mode="xyxy") boxlist.add_field("scores", scores) return boxlist
def select_over_all_levels(self, boxlists): num_images = len(boxlists) results = [] for i in range(num_images): scores = boxlists[i].get_field("scores") labels = boxlists[i].get_field("labels") diagonals = boxlists[i].get_field("diagonals") boxes = boxlists[i].bbox boxlist = boxlists[i] result = [] # skip the background for j in tqdm(range(1, self.num_classes)): inds = (labels == j).nonzero().view(-1) scores_j = scores[inds] boxes_j = boxes[inds, :].view(-1, 4) diagonals_j = diagonals[inds, :].view(-1, 16) boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy") boxlist_for_class.add_field("scores", scores_j) boxlist_for_class.add_field("diagonals", diagonals_j) if scores_j.size()[0] != 0: boxlist_for_class = diagonal_nms(boxlist_for_class, self.nms_thresh, self.nms_topk, score_field="scores") num_labels = len(boxlist_for_class) boxlist_for_class.add_field( "labels", torch.full((num_labels, ), j, dtype=torch.int64, device=scores.device)) result.append(boxlist_for_class) result = cat_boxlist(result) # new_result = all_class_nms(result, self.nms_thresh) number_of_detections = len(result) print('Number of detections in this image {}'.format( number_of_detections)) # Limit to max_per_image detections **over all classes** if number_of_detections > self.fpn_post_nms_top_n > 0: cls_scores = result.get_field("scores") image_thresh, _ = torch.kthvalue( cls_scores.cpu(), number_of_detections - self.fpn_post_nms_top_n + 1) # image_thresh = torch.as_tensor(0.5) keep = cls_scores >= image_thresh.item() keep = torch.nonzero(keep).squeeze(1) result = result[keep] result = all_class_nms(result, self.nms_thresh) results.append(result) return results
def __getitem__(self, item): img = Image.open(self.image_lists[item]).convert("RGB") # dummy target w, h = img.size target = BoxList([[0, 0, w, h]], img.size, mode="xyxy") if self.transforms is not None: img, target = self.transforms(img, target) return img, target
def forward(self, x, boxes, features): """ Arguments: x (Tensor): the attribute logits boxes (list[BoxList]): bounding boxes that are used as reference, one for each image features (Tensor) : attribute features Returns: results (list[BoxList]): one BoxList for each image, containing the extra field attribute """ boxes_per_image = [len(box) for box in boxes] attribute_probs = F.softmax(x, -1) num_classes = attribute_probs.shape[1] attribute_probs = attribute_probs.split(boxes_per_image, dim=0) features = features.split(boxes_per_image, dim=0) results = [] for box, prob, feature in zip(boxes, attribute_probs, features): # copy the current boxes boxlist = BoxList(box.bbox, box.size, mode="xyxy") for field in box.fields(): boxlist.add_field(field, box.get_field(field)) if self.output_feature: boxlist.add_field('attr_feature', feature) # filter out low probability and redundent boxes boxlist = self.filter_results(boxlist, prob, feature, num_classes) results.append(boxlist) return results
def forward(self, x, boxes): """ Arguments: x (Tensor): the mask logits boxes (list[BoxList]): bounding boxes that are used as reference, one for ech image Returns: results (list[BoxList]): one BoxList for each image, containing the extra field mask """ mask_prob = x.sigmoid() # select masks coresponding to the predicted classes num_masks = x.shape[0] labels = [bbox.get_field("labels") for bbox in boxes] labels = torch.cat(labels) index = torch.arange(num_masks, device=labels.device) mask_prob = mask_prob[index, labels][:, None] boxes_per_image = [len(box) for box in boxes] mask_prob = mask_prob.split(boxes_per_image, dim=0) if self.masker: mask_prob = self.masker(mask_prob, boxes) results = [] for prob, box in zip(mask_prob, boxes): bbox = BoxList(box.bbox, box.size, mode="xyxy") for field in box.fields(): bbox.add_field(field, box.get_field(field)) bbox.add_field("mask", prob) results.append(bbox) return results
def forward(self, image_list, feature_maps): grid_sizes = [feature_map.shape[-2:] for feature_map in feature_maps] anchors_over_all_feature_maps = self.grid_anchors(grid_sizes) anchors = [] for i, (image_height, image_width) in enumerate(image_list.image_sizes): anchors_in_image = [] for anchors_per_feature_map in anchors_over_all_feature_maps: boxlist = BoxList( anchors_per_feature_map, (image_width, image_height), mode="xyxy" ) self.add_visibility_to(boxlist) anchors_in_image.append(boxlist) anchors.append(anchors_in_image) return anchors
def prepare_empty_boxlist(self, boxlist): device = boxlist.bbox.device boxlist_empty = BoxList(torch.zeros((0, 4)).to(device), boxlist.size, mode='xyxy') boxlist_empty.add_field("scores", torch.Tensor([]).to(device)) boxlist_empty.add_field( "labels", torch.full((0, ), -1, dtype=torch.int64, device=device)) return boxlist_empty
def select_over_all_levels(self, boxlists): num_images = len(boxlists) results = [] for i in range(num_images): scores = boxlists[i].get_field("scores") labels = boxlists[i].get_field("labels") masks = boxlists[i].get_field("mask") boxes = boxlists[i].bbox boxlist = boxlists[i] result = [] # skip the background for j in range(1, self.num_classes): inds = (labels == j).nonzero().view(-1) scores_j = scores[inds] boxes_j = boxes[inds, :].view(-1, 4) boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy") boxlist_for_class.add_field("scores", scores_j) boxlist_for_class, keep = boxlist_nms_with_keep( boxlist_for_class, self.nms_thresh, score_field="scores") num_labels = len(boxlist_for_class) boxlist_for_class.add_field( "labels", torch.full((num_labels, ), j, dtype=torch.int64, device=scores.device)) boxlist_for_class.add_field("mask", masks[inds[keep], :, :, :]) result.append(boxlist_for_class) result = cat_boxlist(result) number_of_detections = len(result) # Limit to max_per_image detections **over all classes** if number_of_detections > self.fpn_post_nms_top_n > 0: cls_scores = result.get_field("scores") image_thresh, _ = torch.kthvalue( cls_scores.cpu(), number_of_detections - self.fpn_post_nms_top_n + 1) keep = cls_scores >= image_thresh.item() keep = torch.nonzero(keep).squeeze(1) result = result[keep] results.append(result) return results
def get_groundtruth(self, index): anno_name = self.img_item_name[index][:-4]+'.xml' anno_path = os.path.join(self._annodir,anno_name) anno = ET.parse(anno_path).getroot() anno = self._preprocess_annotation(anno) height, width = anno["im_info"] target = BoxList(anno["boxes"], (width, height), mode="xyxy") target.add_field("labels", anno["labels"]) target.add_field("difficult", anno["difficult"]) return target
def get_groundtruth(self, index): img_id = self.ids[index] anno = ET.parse(self._annopath % img_id).getroot() anno = self._preprocess_annotation(anno) height, width = anno["im_info"] try: target = BoxList(anno["boxes"], (width, height), mode="xyxy") except ValueError: raise ValueError("bbox error in {}, {}".format( self._annopath % img_id, anno["boxes"])) else: target.add_field("labels", anno["labels"]) target.add_field("difficult", anno["difficult"]) return target
def merge_result_from_multi_scales(boxlists, nms_type='nms', vote_thresh=0.65): num_images = len(boxlists) results = [] for i in range(num_images): scores = boxlists[i].get_field("scores") labels = boxlists[i].get_field("labels") boxes = boxlists[i].bbox boxlist = boxlists[i] result = [] # skip the background for j in range(1, cfg.MODEL.RETINANET.NUM_CLASSES): inds = (labels == j).nonzero().view(-1) scores_j = scores[inds] boxes_j = boxes[inds, :].view(-1, 4) boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy") boxlist_for_class.add_field("scores", scores_j) boxlist_for_class = boxlist_nms(boxlist_for_class, cfg.MODEL.ATSS.NMS_TH, score_field="scores", nms_type=nms_type, vote_thresh=vote_thresh) num_labels = len(boxlist_for_class) boxlist_for_class.add_field( "labels", torch.full((num_labels, ), j, dtype=torch.int64, device=scores.device)) result.append(boxlist_for_class) result = cat_boxlist(result) number_of_detections = len(result) # Limit to max_per_image detections **over all classes** if number_of_detections > cfg.MODEL.ATSS.PRE_NMS_TOP_N > 0: cls_scores = result.get_field("scores") image_thresh, _ = torch.kthvalue( cls_scores.cpu(), number_of_detections - cfg.MODEL.ATSS.PRE_NMS_TOP_N + 1) keep = cls_scores >= image_thresh.item() keep = torch.nonzero(keep).squeeze(1) result = result[keep] results.append(result) return results
def _py_bbox_list_to_bbox_list(self, py_bbox_list, im_size): ''' :param py_bbox_list: :param im_size: (w, h) :return: ''' bboxes = [] labels = [] scores = [] for item in py_bbox_list: bboxes.append(item["box"]) labels.append(item["label_id"]) scores.append(item["score"]) box_list = BoxList(torch.tensor(bboxes, dtype=torch.float32), im_size) box_list.add_field("labels", torch.tensor(labels, dtype=torch.long)) box_list.add_field("scores", torch.tensor(scores, dtype=torch.float32)) return box_list
def __getitem__(self, idx): # img, anno = super(COCODataset, self).__getitem__(idx) coco = self.coco img_id = self.ids[idx] ann_ids = coco.getAnnIds(imgIds=img_id) anno = coco.loadAnns(ann_ids) path = coco.loadImgs(img_id)[0]['file_name'] img = Image.open(os.path.join(self.root, path)).convert('RGB') # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) # masks = [obj["segmentation"] for obj in anno] # masks = SegmentationMask(masks, img.size, mode='poly') # target.add_field("masks", masks) if anno and "keypoints" in anno[0]: keypoints = [obj["keypoints"] for obj in anno] keypoints = PersonKeypoints(keypoints, img.size) target.add_field("keypoints", keypoints) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, idx
def filter_results(self, boxlist, num_classes): """Returns bounding-box detection results by thresholding on scores and applying non-maximum suppression (NMS). """ # unwrap the boxlist to avoid additional overhead. # if we had multi-class NMS, we could perform this directly on the boxlist boxes = boxlist.bbox.reshape(-1, num_classes * 4) scores = boxlist.get_field("scores").reshape(-1, num_classes) device = scores.device result = [] # Apply threshold on detection probabilities and apply NMS # Skip j = 0, because it's the background class inds_all = scores > self.score_thresh for j in range(1, num_classes): inds = inds_all[:, j].nonzero().squeeze(1) scores_j = scores[inds, j] boxes_j = boxes[inds, j * 4:(j + 1) * 4] boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy") boxlist_for_class.add_field("scores", scores_j) boxlist_for_class = boxlist_nms(boxlist_for_class, self.nms) num_labels = len(boxlist_for_class) boxlist_for_class.add_field( "labels", torch.full((num_labels, ), j, dtype=torch.int64, device=device)) result.append(boxlist_for_class) result = cat_boxlist(result) number_of_detections = len(result) # Limit to max_per_image detections **over all classes** if number_of_detections > self.detections_per_img > 0: cls_scores = result.get_field("scores") image_thresh, _ = torch.kthvalue( cls_scores.cpu(), number_of_detections - self.detections_per_img + 1) keep = cls_scores >= image_thresh.item() keep = torch.nonzero(keep).squeeze(1) result = result[keep] return result
def forward(self, x, boxes): mask_prob = x scores = None if self.keypointer: mask_prob, scores = self.keypointer(x, boxes) assert len(boxes) == 1, "Only non-batched inference supported for now" boxes_per_image = [box.bbox.size(0) for box in boxes] mask_prob = mask_prob.split(boxes_per_image, dim=0) scores = scores.split(boxes_per_image, dim=0) results = [] for prob, box, score in zip(mask_prob, boxes, scores): bbox = BoxList(box.bbox, box.size, mode="xyxy") for field in box.fields(): bbox.add_field(field, box.get_field(field)) prob = PersonKeypoints(prob, box.size) prob.add_field("logits", score) bbox.add_field("keypoints", prob) results.append(bbox) return results
def evaluate_box_proposals(predictions, dataset, thresholds=None, area="all", limit=None): """Evaluate detection proposal recall metrics. This function is a much faster alternative to the official COCO API recall evaluation code. However, it produces slightly different results. """ # Record max overlap value for each gt box # Return vector of overlap values areas = { "all": 0, "small": 1, "medium": 2, "large": 3, "96-128": 4, "128-256": 5, "256-512": 6, "512-inf": 7, } area_ranges = [ [0**2, 1e5**2], # all [0**2, 32**2], # small [32**2, 96**2], # medium [96**2, 1e5**2], # large [96**2, 128**2], # 96-128 [128**2, 256**2], # 128-256 [256**2, 512**2], # 256-512 [512**2, 1e5**2], ] # 512-inf assert area in areas, "Unknown area range: {}".format(area) area_range = area_ranges[areas[area]] gt_overlaps = [] num_pos = 0 for image_id, prediction in enumerate(predictions): original_id = dataset.id_to_img_map[image_id] img_info = dataset.get_img_info(image_id) image_width = img_info["width"] image_height = img_info["height"] prediction = prediction.resize((image_width, image_height)) # sort predictions in descending order # TODO maybe remove this and make it explicit in the documentation inds = prediction.get_field("objectness").sort(descending=True)[1] prediction = prediction[inds] ann_ids = dataset.coco.getAnnIds(imgIds=original_id) anno = dataset.coco.loadAnns(ann_ids) gt_boxes = [obj["bbox"] for obj in anno if obj["iscrowd"] == 0] gt_boxes = torch.as_tensor(gt_boxes).reshape( -1, 4) # guard against no boxes gt_boxes = BoxList(gt_boxes, (image_width, image_height), mode="xywh").convert("xyxy") gt_areas = torch.as_tensor( [obj["area"] for obj in anno if obj["iscrowd"] == 0]) if len(gt_boxes) == 0: continue valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) gt_boxes = gt_boxes[valid_gt_inds] num_pos += len(gt_boxes) if len(gt_boxes) == 0: continue if len(prediction) == 0: continue if limit is not None and len(prediction) > limit: prediction = prediction[:limit] overlaps = boxlist_iou(prediction, gt_boxes) _gt_overlaps = torch.zeros(len(gt_boxes)) for j in range(min(len(prediction), len(gt_boxes))): # find which proposal box maximally covers each gt box # and get the iou amount of coverage for each gt box max_overlaps, argmax_overlaps = overlaps.max(dim=0) # find which gt box is 'best' covered (i.e. 'best' = most iou) gt_ovr, gt_ind = max_overlaps.max(dim=0) assert gt_ovr >= 0 # find the proposal box that covers the best covered gt box box_ind = argmax_overlaps[gt_ind] # record the iou coverage of this gt box _gt_overlaps[j] = overlaps[box_ind, gt_ind] assert _gt_overlaps[j] == gt_ovr # mark the proposal box and the gt box as used overlaps[box_ind, :] = -1 overlaps[:, gt_ind] = -1 # append recorded iou coverage level gt_overlaps.append(_gt_overlaps) gt_overlaps = torch.cat(gt_overlaps, dim=0) gt_overlaps, _ = torch.sort(gt_overlaps) if thresholds is None: step = 0.05 thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) recalls = torch.zeros_like(thresholds) # compute recall for each iou threshold for i, t in enumerate(thresholds): recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) # ar = 2 * np.trapz(recalls, thresholds) ar = recalls.mean() return { "ar": ar, "recalls": recalls, "thresholds": thresholds, "gt_overlaps": gt_overlaps, "num_pos": num_pos, }
def main(args): annFile = 'datasets/coco/annotations/instances_train2017_0.5.json' coco = COCO(annFile) with open(annFile, 'r') as f: result_json = json.load(f) annos_json = result_json['annotations'] # anno_id = max([ann['id'] for ann in annos_json]) + 1 output_dir = os.path.join(args.predictions, 'coco_2017_train_partial') image_ids = torch.load(os.path.join(output_dir, 'image_ids.pth')) predictions = torch.load(os.path.join(output_dir, 'predictions.pth')) anno_id = max(torch.load(os.path.join(output_dir, 'box_ids.pth'))) + 1 imgIds = sorted(coco.getImgIds()) threshold = args.confidence # threshold = torch.tensor([-1.0, 0.46633365750312805, 0.4409848749637604, 0.47267603874206543, 0.4707889258861542, 0.5220812559127808, 0.5358721613883972, 0.5226702690124512, 0.45160290598869324]) iou_threshold = 0.5 cpu_device = torch.device("cpu") partial_box_num = 0 N = len(image_ids) for i in tqdm(range(N)): im_idx = image_ids[i] bbox = predictions[i] imginfo = coco.loadImgs(imgIds[im_idx])[0] image_width = imginfo['width'] image_height = imginfo['height'] # load annotations partial_anns = coco.loadAnns(coco.getAnnIds(imgIds=(imgIds[im_idx], ))) # full_anns = coco_full.loadAnns(coco_full.getAnnIds(imgIds=(imgIds[im_idx],), catIds=catIds)) partial_boxes = [obj["bbox"] for obj in partial_anns] partial_boxes_ids = set([obj["id"] for obj in partial_anns]) partial_boxes = torch.as_tensor(partial_boxes).reshape( -1, 4) # guard against no boxes partial_boxes = BoxList(partial_boxes, (image_width, image_height), mode="xywh").convert("xyxy") partial_box_num += len(partial_boxes_ids) # get predictions bbox = bbox.resize((image_width, image_height)) bbox = bbox.to(cpu_device) # generate pseudo labels idx = generate_pseudo_label_with_confidence_score( bbox, im_idx, threshold) if len(idx) > 0: pseudo_labels = bbox[idx] scores = pseudo_labels.get_field("scores").tolist() # compute iou overlaps = boxlist_iou(partial_boxes, pseudo_labels) matched_id = [True] * len(pseudo_labels) # remove predictions for partial labels for i in range(len(partial_boxes)): matched = np.argmax(overlaps[i]) if overlaps[i, matched] >= iou_threshold: matched_id[matched] = False pseudo_labels = pseudo_labels[matched_id] # print(num, len(pseudo_labels)) pseudo_annos, anno_id = new_annotation_json( pseudo_labels, imgIds[im_idx], anno_id) annos_json.extend(pseudo_annos) print('confidence threshold: {}'.format(threshold)) result_json['annotations'] = annos_json with open(args.annotation, 'w') as f: json.dump(result_json, f) print(partial_box_num, len(result_json['annotations']))
def forward_for_single_feature_map( self, locations, box_cls, box_regression, centerness, image_sizes): """ Arguments: anchors: list[BoxList] box_cls: tensor of size N, A * C, H, W box_regression: tensor of size N, A * 4, H, W """ N, C, H, W = box_cls.shape # put in the same format as locations box_cls = box_cls.view(N, C, H, W).permute(0, 2, 3, 1) box_cls = box_cls.reshape(N, -1, C).sigmoid() box_regression = box_regression.view(N, 4, H, W).permute(0, 2, 3, 1) box_regression = box_regression.reshape(N, -1, 4) centerness = centerness.view(N, 1, H, W).permute(0, 2, 3, 1) centerness = centerness.reshape(N, -1).sigmoid() candidate_inds = box_cls > self.pre_nms_thresh pre_nms_top_n = candidate_inds.view(N, -1).sum(1) pre_nms_top_n = pre_nms_top_n.clamp(max=self.pre_nms_top_n) # multiply the classification scores with centerness scores box_cls = box_cls * centerness[:, :, None] results = [] for i in range(N): per_box_cls = box_cls[i] per_candidate_inds = candidate_inds[i] per_box_cls = per_box_cls[per_candidate_inds] per_candidate_nonzeros = per_candidate_inds.nonzero() per_box_loc = per_candidate_nonzeros[:, 0] per_class = per_candidate_nonzeros[:, 1] + 1 per_box_regression = box_regression[i] per_box_regression = per_box_regression[per_box_loc] per_locations = locations[per_box_loc] per_pre_nms_top_n = pre_nms_top_n[i] if per_candidate_inds.sum().item() > per_pre_nms_top_n.item(): per_box_cls, top_k_indices = \ per_box_cls.topk(per_pre_nms_top_n, sorted=False) per_class = per_class[top_k_indices] per_box_regression = per_box_regression[top_k_indices] per_locations = per_locations[top_k_indices] detections = torch.stack([ per_locations[:, 0] - per_box_regression[:, 0], per_locations[:, 1] - per_box_regression[:, 1], per_locations[:, 0] + per_box_regression[:, 2], per_locations[:, 1] + per_box_regression[:, 3], ], dim=1) h, w = image_sizes[i] boxlist = BoxList(detections, (int(w), int(h)), mode="xyxy") boxlist.add_field("labels", per_class) boxlist.add_field("scores", torch.sqrt(per_box_cls)) boxlist = boxlist.clip_to_image(remove_empty=False) boxlist = remove_small_boxes(boxlist, self.min_size) results.append(boxlist) return results
def forward_for_mask(self, boxlists, pixel_embed): N, dim, m_h, m_w = pixel_embed.shape new_boxlists = [] stride = self.fpn_strides[0] / self.mask_scale_factor for im in range(N): boxlist = boxlists[im] boxes = boxlist.bbox input_w, input_h = boxlist.size proposal_embed = boxlist.get_field('proposal_embed') if len(proposal_embed) == 0: new_boxlist = BoxList(boxes, boxlist.size, mode="xyxy") new_boxlist.add_field("labels", boxlist.get_field("labels")) new_boxlist.add_field("scores", boxlist.get_field("scores")) new_boxlist.add_field('mask', torch.tensor([])) if self.post_process_masks: new_boxlist.add_field('stride', torch.tensor(1)) new_boxlist.add_field('mask_th', torch.tensor(0.0)) else: new_boxlist.add_field('stride', torch.tensor(stride)) new_boxlist.add_field('mask_th', torch.tensor(self.mask_th)) new_boxlists.append(new_boxlist) continue mask_boxes = boxes / stride box_masks = boxes_to_masks(mask_boxes, m_h, m_w) proposal_margin = boxlist.get_field('proposal_margin') mask_prob = self.compute_mask_prob(pixel_embed[im], proposal_embed, proposal_margin, mask_boxes) masks = mask_prob * box_masks.float() if self.post_process_masks: masks = torch.nn.functional.interpolate( input=masks.unsqueeze(1).float(), scale_factor=stride, mode="bilinear", align_corners=False).gt(self.mask_th) masks = masks[:, 0, :input_h, :input_w] new_boxlist = BoxList(boxes, boxlist.size, mode="xyxy") new_boxlist.add_field('mask', masks) new_boxlist.add_field("labels", boxlist.get_field("labels")) new_boxlist.add_field("scores", boxlist.get_field("scores")) if self.post_process_masks: new_boxlist.add_field('stride', torch.tensor(1)) new_boxlist.add_field('mask_th', torch.tensor(0.0)) else: new_boxlist.add_field('stride', torch.tensor(stride)) new_boxlist.add_field('mask_th', torch.tensor(self.mask_th)) new_boxlists.append(new_boxlist) return new_boxlists
def __call__(self, image, target=None): if self.to_bgr255: image = image[[2, 1, 0]] * 255 image = F.normalize(image, mean=self.mean, std=self.std) if target is None: return image return image, target if __name__ == '__main__': import cv2 from fcos_core.structures.bounding_box import BoxList image = Image.open("/home/nie/lena.jpg").convert("RGB") target = BoxList(torch.tensor([[80, 92, 152, 194]], dtype=torch.float32), image.size, mode="xyxy") target.add_field("labels", torch.tensor([1])) transform = Compose([ # Resize(256, 256), # RandomHorizontalFlip(0.5), ImgJitter(), ImgAug_Private() ]) for i in range(10): _image, _target = transform(image=image, target=target) rgb_image = np.array(_image) bboxes = np.array(_target.bbox.numpy(), dtype=np.int).tolist() for bbox in bboxes: rgb_image = cv2.rectangle(rgb_image, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), 2)
def forward_for_single_feature_map(self, anchors, box_cls, box_regression): """ Arguments: anchors: list[BoxList] box_cls: tensor of size N, A * C, H, W box_regression: tensor of size N, A * 4, H, W """ device = box_cls.device N, _, H, W = box_cls.shape A = box_regression.size(1) // 4 C = box_cls.size(1) // A # put in the same format as anchors box_cls = permute_and_flatten(box_cls, N, A, C, H, W) box_cls = box_cls.sigmoid() box_regression = permute_and_flatten(box_regression, N, A, 4, H, W) box_regression = box_regression.reshape(N, -1, 4) num_anchors = A * H * W candidate_inds = box_cls > self.pre_nms_thresh pre_nms_top_n = candidate_inds.view(N, -1).sum(1) pre_nms_top_n = pre_nms_top_n.clamp(max=self.pre_nms_top_n) results = [] for per_box_cls, per_box_regression, per_pre_nms_top_n, \ per_candidate_inds, per_anchors in zip( box_cls, box_regression, pre_nms_top_n, candidate_inds, anchors): # Sort and select TopN # TODO most of this can be made out of the loop for # all images. # TODO:Yang: Not easy to do. Because the numbers of detections are # different in each image. Therefore, this part needs to be done # per image. per_box_cls = per_box_cls[per_candidate_inds] per_box_cls, top_k_indices = \ per_box_cls.topk(per_pre_nms_top_n, sorted=False) per_candidate_nonzeros = \ per_candidate_inds.nonzero()[top_k_indices, :] per_box_loc = per_candidate_nonzeros[:, 0] per_class = per_candidate_nonzeros[:, 1] per_class += 1 detections = self.box_coder.decode( per_box_regression[per_box_loc, :].view(-1, 4), per_anchors.bbox[per_box_loc, :].view(-1, 4)) boxlist = BoxList(detections, per_anchors.size, mode="xyxy") boxlist.add_field("labels", per_class) boxlist.add_field("scores", per_box_cls) boxlist = boxlist.clip_to_image(remove_empty=False) boxlist = remove_small_boxes(boxlist, self.min_size) results.append(boxlist) return results
def __call__(self, tensor, target): if self.ratio >= 1.0: return tensor, target self.img_pool.append({'tensor': tensor, 'target': target}) if len(self.img_pool) > self.img_pool_size: self.img_pool.pop(0) c, h, w = tensor.shape if self.size_divisible > 0: h = int(math.ceil(h / self.size_divisible) * self.size_divisible) w = int(math.ceil(w / self.size_divisible) * self.size_divisible) new_h, new_w = int(self.ratio * h), int(self.ratio * w) in_tensor, in_target = scale_jitter(tensor, target, (new_h, new_w)) if len(self.img_pool) < 4: tensor_out, target_out = in_tensor, in_target else: pad_imgs = random.sample(self.img_pool, 3) pad_tensors, pad_targets = [], [] for img in pad_imgs: pad_tensor, pad_target = scale_jitter(img['tensor'], img['target'], (new_h, new_w)) pad_tensors.append(pad_tensor) pad_targets.append(pad_target) crop_boxes = [(0, 0, w - new_w, new_h), (0, 0, new_w, h - new_h), (0, 0, w - new_w, h - new_h)] tensor_out = in_tensor.new(*(c, h, w)).zero_() tensor_out[:c, :new_h, :new_w].copy_(in_tensor) tensor_out[:c, :new_h, new_w:].copy_( pad_tensors[0][:c, :crop_boxes[0][3], :crop_boxes[0][2]]) tensor_out[:c, new_h:, :new_w].copy_( pad_tensors[1][:c, :crop_boxes[1][3], :crop_boxes[1][2]]) tensor_out[:c, new_h:, new_w:].copy_( pad_tensors[2][:c, :crop_boxes[2][3], :crop_boxes[2][2]]) crop_targets = [] for i, pad_target in enumerate(pad_targets): crop_target = pad_target.crop(crop_boxes[i]) ious = crop_target.area() / pad_target.area() crop_target = crop_target[ious >= self.iou_threshold] crop_targets.append(crop_target) offsets_box = [ torch.Tensor([0.0, 0.0, 0.0, 0.0]), torch.Tensor([new_w, 0.0, new_w, 0.0]), torch.Tensor([0.0, new_h, 0.0, new_h]), torch.Tensor([new_w, new_h, new_w, new_h]) ] target_out = torch.cat([ target.bbox + offsets_box[i] for i, target in enumerate([in_target] + crop_targets) ], dim=0) target_out = BoxList(target_out, (w, h), mode='xyxy') target_out.add_field( 'labels', torch.cat([ target.extra_fields['labels'] for target in ([in_target] + crop_targets) ], dim=-1)) polys_list = [[ poly.polygons[0] for poly in target.extra_fields['masks'].instances.polygons ] for target in ([in_target] + crop_targets)] offsets_mask = [[0.0, 0.0], [new_w, 0.0], [0.0, new_h], [new_w, new_h]] syn_mask = [] for i, polys in enumerate(polys_list): syn_mask += [[ list( np.array(poly) + np.array(offsets_mask[i] * int(len(poly) / 2))) ] for poly in polys] syn_mask = SegmentationMask(syn_mask, (w, h), mode='poly') target_out.add_field('masks', syn_mask) return tensor_out, target_out