def __getitem__(self, idx): img, anno = super(COCODataset, self).__getitem__(idx) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size) target.add_field("masks", masks) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, idx
def select_over_all_levels(self, boxlists): num_images = len(boxlists) results = [] for i in range(num_images): scores = boxlists[i].get_field("scores") labels = boxlists[i].get_field("labels") boxes = boxlists[i].bbox boxlist = boxlists[i] result = [] # skip the background for j in range(1, self.num_classes): inds = (labels == j).nonzero().view(-1) scores_j = scores[inds] boxes_j = boxes[inds, :].view(-1, 4) boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy") boxlist_for_class.add_field("scores", scores_j) boxlist_for_class = boxlist_nms(boxlist_for_class, self.nms_thresh, score_field="scores") num_labels = len(boxlist_for_class) boxlist_for_class.add_field( "labels", torch.full((num_labels, ), j, dtype=torch.int64, device=scores.device)) result.append(boxlist_for_class) result = cat_boxlist(result) number_of_detections = len(result) # Limit to max_per_image detections **over all classes** if number_of_detections > self.fpn_post_nms_top_n > 0: cls_scores = result.get_field("scores") image_thresh, _ = torch.kthvalue( cls_scores.cpu(), number_of_detections - self.fpn_post_nms_top_n + 1) keep = cls_scores >= image_thresh.item() keep = torch.nonzero(keep).squeeze(1) result = result[keep] results.append(result) return results
def __getitem__(self, index): img_id = self.ids[index] im_path = os.path.join(self.root, img_id + '.jpg') img = Image.open(im_path).convert("RGB") im = cv2.imread(im_path) anno = self.get_groundtruth(index) anno["im_info"] = [im.shape[0], im.shape[1]] height, width = anno["im_info"] target = BoxList(anno["boxes"], (width, height), mode="xyxy") target.add_field("labels", anno["labels"]) target.add_field("difficult", anno["difficult"]) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, index
def remove_low_score_boxes(self, boxlist, num_classes): boxes = boxlist.bbox scores = boxlist.get_field("scores") device = scores.device num_boxes = int(boxlist.bbox.shape[0] / num_classes) box_labels = torch.arange(num_classes).repeat(num_boxes).to(device) keep_inds = (box_labels > 0) * (scores > self.score_thresh) boxes = boxes[keep_inds, :] box_labels = box_labels[keep_inds] scores = scores[keep_inds] boxelist_filtered = BoxList(boxes, boxlist.size, "xyxy") boxelist_filtered.add_field("scores", scores) boxelist_filtered.add_field("box_labels", box_labels) return boxelist_filtered
def forward(self, x, boxes): """ Arguments: x (Tensor): the mask logits boxes (list[BoxList]): bounding boxes that are used as reference, one for ech image Returns: results (list[BoxList]): one BoxList for each image, containing the extra field mask """ mask_prob = x.sigmoid() # select masks coresponding to the predicted classes num_masks = x.shape[0] labels = [bbox.get_field("labels") for bbox in boxes] # the id labels labels = torch.cat(labels) index = torch.arange(num_masks, device=labels.device) # the predict boxes mask_prob = mask_prob[ index, labels][:, None] # select the mask of the box with label boxes_per_image = [len(box) for box in boxes] mask_prob = mask_prob.split(boxes_per_image, dim=0) if self.masker: # height, width = images.tensors.shape[-2:] # boxes_init = [] # for i in boxes: # boxes_init.append(i.resize((width, height))) # mask_prob = self.masker(mask_prob, boxes_init) mask_prob = self.masker(mask_prob, boxes) results = [] for prob, box in zip(mask_prob, boxes): bbox = BoxList(box.bbox, box.size, mode="xyxy") for field in box.fields(): bbox.add_field(field, box.get_field(field)) bbox.add_field("mask", prob) results.append(bbox) return results
def prepare_boxlist(boxes, scores, image_shape, ids): """ Returns BoxList from `boxes` and adds probability scores information as an extra field `boxes` has shape (#detections, 4 * #classes), where each row represents a list of predicted bounding boxes for each of the object classes in the dataset (including the background class). The detections in each row originate from the same object proposal. `scores` has shape (#detection, #classes), where each row represents a list of object detection confidence scores for each of the object classes in the dataset (including the background class). `scores[i, j]`` corresponds to the box at `boxes[i, j * 4:(j + 1) * 4]`. """ boxes = boxes.reshape(-1, 4) scores = scores.reshape(-1) boxlist = BoxList(boxes, image_shape, mode="xyxy") boxlist.add_field("scores", scores) boxlist.add_field("ids", ids) return boxlist
def filter_results(self, boxlist, num_classes): """Returns bounding-box detection results by thresholding on scores and applying non-maximum suppression (NMS). """ # unwrap the boxlist to avoid additional overhead. # if we had multi-class NMS, we could perform this directly on the boxlist boxes = boxlist.bbox.reshape(-1, num_classes * 4) scores = boxlist.get_field("scores").reshape(-1, num_classes) device = scores.device result = [] # Apply threshold on detection probabilities and apply NMS # Skip j = 0, because it's the background class inds_all = scores > self.score_thresh for j in range(1, num_classes): inds = inds_all[:, j].nonzero().squeeze(1) scores_j = scores[inds, j] boxes_j = boxes[inds, j * 4:(j + 1) * 4] boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy") boxlist_for_class.add_field("scores", scores_j) boxlist_for_class = boxlist_nms(boxlist_for_class, self.nms, score_field="scores") num_labels = len(boxlist_for_class) boxlist_for_class.add_field( "labels", torch.full((num_labels, ), j, dtype=torch.int64, device=device)) result.append(boxlist_for_class) result = cat_boxlist(result) number_of_detections = len(result) # Limit to max_per_image detections **over all classes** if number_of_detections > self.detections_per_img > 0: cls_scores = result.get_field("scores") image_thresh, _ = torch.kthvalue( cls_scores.cpu(), number_of_detections - self.detections_per_img + 1) keep = cls_scores >= image_thresh.item() keep = torch.nonzero(keep).squeeze(1) result = result[keep] return result
def __call__(self, image, target=None): if target == None: return image if not self.do: return image, target w, h = image.size assert w == h cx = w / 2 cy = cx degree = random.uniform(0, 360) radian = degree * math.pi / 180 new_image = image.rotate(-degree) sin = math.sin(radian) cos = math.cos(radian) masks = target.get_field("masks") polygons = list(map(lambda x: x.polygons[0], masks.instances.polygons)) polygons = torch.stack(polygons, 0).reshape((-1, 2)).t() M = torch.Tensor([[cos, -sin], [sin, cos]]) b = torch.Tensor([[(1 - cos) * cx + cy * sin], [(1 - cos) * cy - cx * sin]]) new_points = M.mm(polygons) + b new_points = new_points.t().reshape((-1, 8)) xmins, _ = torch.min(new_points[:, ::2], 1) ymins, _ = torch.min(new_points[:, 1::2], 1) xmaxs, _ = torch.max(new_points[:, ::2], 1) ymaxs, _ = torch.max(new_points[:, 1::2], 1) boxes = torch.stack([xmins, ymins, xmaxs, ymaxs], 1).reshape((-1, 4)) new_target = BoxList(boxes, image.size, mode="xyxy") new_target._copy_extra_fields(target) new_masks = SegmentationMask(new_points.reshape((-1, 1, 8)).tolist(), image.size, mode='poly') new_target.add_field("masks", new_masks) return new_image, new_target
def __getitem__(self, index): """ Args: index (int): Index Returns: tuple: Tuple (image, target). target is a list of captions for the image. """ img_id, sent_id = self.ids[index].split( '\t')[0], self.ids[index].split('\t')[1] topN_box = self.topN_box_anno[img_id][int(sent_id)] filename = os.path.join(self.img_root, img_id + '.jpg') img = Image.open(filename).convert('RGB') sent_sg = self.sg_anno[img_id]['relations'][int(sent_id)] _, feature_map, precompute_bbox, img_scale, precompute_score, cls_label = self.get_precompute_img_feat( img_id) precompute_bbox = BoxList(precompute_bbox, img.size, mode='xyxy') if cfg.MODEL.VG.USE_BOTTOMUP_NMS: precompute_bbox.add_field("scores", torch.FloatTensor(precompute_score)) precompute_bbox, keep_inds = boxlist_nms( precompute_bbox, cfg.MODEL.VG.BOTTOMUP_NMS_THRESH, require_keep_idx=True) precompute_score = precompute_score[keep_inds.numpy()] sentence = self.get_sentence(img_id, int(sent_id)) phrase_ids, gt_boxes = self.get_gt_boxes(img_id) target = BoxList(gt_boxes, img.size, mode="xyxy") vocab_label_elmo = self.vocab_embed[cls_label] if self.transforms is not None: img, target, precompute_bbox, img_scale = self.transforms( img, target, precompute_bbox, img_scale) return None, target, img_id, phrase_ids, sent_id, sentence, precompute_bbox, precompute_score, feature_map, vocab_label_elmo, sent_sg, topN_box
def __getitem__(self, idx): # use zipreader, change the function of super.getitem coco = self.coco img_id = self.ids[idx] ann_ids = coco.getAnnIds(imgIds=img_id) anno = coco.loadAnns(ann_ids) path = coco.loadImgs(img_id)[0]['file_name'] img = Image.open(os.path.join(self.root, path)).convert('RGB') # In philly cluster use zipreader instead Image.open # img = zipreader.imread(os.path.join(self.root, path), \ # cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) # img = Image.fromarray(img) # img = cv2.imread(os.path.join(self.root, path), cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) # img = Image.fromarray(img) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size) target.add_field("masks", masks) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, idx
def __getitem__(self, idx): if self.mode == 0: img, anno = super(COCODataset, self).__getitem__(idx) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size) target.add_field("masks", masks) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, idx elif self.mode == 1: img_id = self.ids[index] feature_list = torch.load(self._split_feature % (self.backbone, self.resolution, img_id) , map_location=torch.device("cpu")) if self.special_deal: label = torch.load(self._split_label % (self.backbone, self.resolution, img_id) , map_location=torch.device("cpu")) reg = torch.load(self._split_reg % (self.backbone, self.resolution, img_id) , map_location=torch.device("cpu")) return feature_list, label, reg, index else: target = torch.load(self._split_target % (self.backbone, self.resolution, img_id) , map_location=torch.device("cpu")) return feature_list, target, index else: raise ValueError("Mode {} do not support now".format(self.mode))
def convert_given_detections_to_boxlist(entities: [AnnoEntity], video_width, video_height, class_table=None): # default class is person only if class_table is None: class_table = ["person"] boxes = [_entity.bbox for _entity in entities] boxes = torch.as_tensor(boxes).reshape(-1, 4) _labels = [class_table.index(list(_entity.labels.keys())[0]) + 1 for _entity in entities] _labels = torch.tensor(_labels, dtype=torch.int64) _scores = torch.tensor([_entity.confidence for _entity in entities]) _ids = torch.tensor([-1 for _entity in entities], dtype=torch.int64) boxlist = BoxList(boxes, [video_width, video_height], mode='xywh').convert('xyxy') boxlist.add_field('labels', _labels) boxlist.add_field('scores', _scores) boxlist.add_field('ids', _ids) return boxlist
def json2boxlist(imgs): results = {} for img, boxs in imgs.items(): i = Image.open(os.path.join(img_dir, img)) boxes = [obj["bbox"] for obj in boxs] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, i.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in boxs] # classes = [c for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) scores = [obj['score'] for obj in boxs] scores = torch.tensor(scores) target.add_field("scores", scores) results[img] = target return results
def _get_instance_box_list(self, img, anno, image_id): # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") # assume these are consistent. classes = [obj["category_id"] for obj in anno] classes = [self.json_thing_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size) target.add_field("masks", masks) return target
def forward(self, boxes, pred_maskiou, labels): num_masks = pred_maskiou.shape[0] index = torch.arange(num_masks, device=labels.device) maskious = pred_maskiou[index, labels] maskious = [maskious] results = [] count = 0 # for maskiou, box in zip(maskious, boxes): for box in boxes: bbox = BoxList(box.bbox, box.size, mode="xyxy") for field in box.fields(): bbox.add_field(field, box.get_field(field)) bbox_scores = bbox.get_field("scores") mask_scores = bbox_scores * maskious[0][count:count + len(bbox_scores)] bbox.add_field("mask_scores", mask_scores) results.append(bbox) count += len(bbox_scores) return results
def __getitem__(self, idx): image, anno = self.pull_item(idx) labels = torch.from_numpy(np.array(anno['labels'])) # create a BoxList from the boxes # image need to be a PIL Image boxlist = BoxList(anno['bndboxes'], image.size, mode="xyxy") # add the labels to the boxlist boxlist.add_field("labels", labels) # add masks if anno and "polygons" in anno: masks = SegmentationMask(anno['polygons'], image.size, mode='poly') boxlist.add_field("masks", masks) if self._transforms: image, boxlist = self._transforms(image, boxlist) # return the image, the boxlist and the idx in your dataset return image, boxlist, idx
def get_annotation(self, image_id): coco = self.coco ann_ids = coco.getAnnIds(imgIds=image_id) img_data = self.coco.imgs[image_id] anno = coco.loadAnns(ann_ids) boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, (img_data['width'], img_data['height']), mode="xywh").convert("xyxy") labels = [obj["category_id"] for obj in anno] labels = [self.json_category_id_to_contiguous_id[c] for c in labels] target.add_field("labels", torch.tensor(labels)) target = target.clip_to_image(remove_empty=True) return { 'boxes': target.bbox.tolist(), 'labels': target.get_field('labels').tolist() }
def __getitem__(self, idx): img_path = self.frames_list[idx] self.img = Image.open(open(img_path, 'rb')) boxes, labels = self.get_groundtruth(img_path) boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, self.img.size, mode="xyxy") # create a BoxList from the boxes # add the labels to the boxlist classes = [self.class_to_ind['tire'] for label in labels] classes = torch.tensor(classes) target.add_field("labels", classes) target = target.clip_to_image(remove_empty=True) if self.transforms: self.img, target = self.transforms(self.img, target) return self.img, target, idx
def __getitem__(self, index): image_id = self.images[index]['id'] img_path = os.path.join(self.images_dir, self.images[index]['file_name']) img = Image.open(img_path).convert("RGB") width, height = img.size[0], img.size[1] boxes = [] labels = [] ann = self.annotations[image_id] for category, x, y, w, h in ann: boxes.append([x, y, x + w, y + h]) labels.append(category) target = BoxList(torch.tensor(boxes, dtype=torch.float32), (width, height), mode="xyxy") target.add_field('labels', torch.tensor(labels)) target = target.clip_to_image(remove_empty=True) if self.transforms is not None: img, target = self.transforms(img, target) return img, target, index
def forward_for_single_feature_map(self, anchors, objectness, box_regression): """ Arguments: anchors: list[BoxList] objectness: tensor of size N, A, H, W box_regression: tensor of size N, A * 4, H, W """ device = objectness.device N, A, H, W = objectness.shape objectness, topk_idx, box_regression = self.objectness_top_k( objectness, box_regression) batch_idx = torch.arange(N, device=device)[:, None] box_regression = box_regression[batch_idx, topk_idx] image_shapes = [box.size for box in anchors] concat_anchors = torch.cat([a.bbox for a in anchors], dim=0) concat_anchors = concat_anchors.reshape(N, -1, 4)[batch_idx, topk_idx] proposals = self.box_coder.decode(box_regression.view(-1, 4), concat_anchors.view(-1, 4)) proposals = proposals.view(N, -1, 4) result = [] for proposal, score, im_shape in zip(proposals, objectness, image_shapes): boxlist = BoxList(proposal, im_shape, mode="xyxy") boxlist.add_field("objectness", score) boxlist = boxlist.clip_to_image(remove_empty=False) boxlist = remove_small_boxes(boxlist, self.min_size) boxlist = boxlist_nms( boxlist, self.nms_thresh, max_proposals=self.post_nms_top_n, score_field="objectness", ) result.append(boxlist) return result
def get_groundtruth(self, idx): bboxes = [] masks = [] svg_name = self.data_dir / self.split / "{:s}.svg".format( self.ids[idx]) with svg_name.open('r') as f_svg: svg = f_svg.read() num_paths = svg.count('polyline') for i in range(1, num_paths + 1): svg_xml = et.fromstring(svg) svg_xml[1] = svg_xml[i] del svg_xml[2:] svg_one = et.tostring(svg_xml, method='xml') # leave only one path y_png = cairosvg.svg2png(bytestring=svg_one) y_img = Image.open(io.BytesIO(y_png)) mask = (np.array(y_img)[:, :, 3] > 0) try: bboxes.append(self.get_bbox(mask, "xyxy")) except: continue masks.append(mask.astype(np.uint8)) # if there is completely no boxes, add dummy boxes and masks if len(bboxes) == 0: print(self.ids[idx], et.tostring(svg_xml, method='xml')) dummy_mask = np.zeros(mask.shape, dtype=np.uint8) dummy_mask[0:4, 0:4] = 1 masks.append(dummy_mask.astype(np.uint8)) bboxes.append(self.get_bbox(dummy_mask, "xyxy")) image_size_dict = self.get_img_info(idx) image_size = (image_size_dict["height"], image_size_dict["width"]) boxlist = BoxList(bboxes, image_size, mode="xyxy") boxlist.add_field("labels", torch.tensor([1] * len(bboxes))) boxlist.add_field("mask", SegmentationMask(masks, image_size, "mask")) return self.svg_to_png(svg), boxlist
def forward(self, x, boxes): """ Arguments: x (Tensor): the mask logits boxes (list[BoxList]): bounding boxes that are used as reference, one for each image Returns: results (list[BoxList]): one BoxList for each image, containing the extra field mask """ if x.device.type == 'mlu': mask_prob = x.sigmoid() bbox = boxes[0] label = boxes[1] score = boxes[2] return (mask_prob, bbox, label, score) mask_prob = x.sigmoid() # select masks coresponding to the predicted classes num_masks = x.shape[0] labels = [bbox.get_field("labels") for bbox in boxes] labels = torch.cat(labels) index = torch.arange(num_masks, device=labels.device) mask_prob = mask_prob[index, labels][:, None] boxes_per_image = [len(box) for box in boxes] mask_prob = mask_prob.split(boxes_per_image, dim=0) if self.masker: mask_prob = self.masker(mask_prob, boxes) results = [] for prob, box in zip(mask_prob, boxes): bbox = BoxList(box.bbox, box.size, mode="xyxy") for field in box.fields(): bbox.add_field(field, box.get_field(field)) bbox.add_field("mask", prob) results.append(bbox) return results
def build_boxlist(fname, single_block=False): with open(fname, 'rb') as f: xml = et.fromstring(f.read()) # image dimensions width, height = (int(xml.attrib[x]) for x in ['width', 'height']) def xyxy(elt): return [float(elt.attrib[x]) for x in 'ltrb'] boxes = [xyxy(elt) for elt in xml.findall('.//block')] if single_block: boxes = boxes[:1] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, (width, height), mode="xyxy") classes = torch.tensor([1]*len(boxes), dtype=torch.int32) target.add_field('labels', classes) return target
def prepare_boxlist(self, boxes, scores, image_shape, cyclic=False, target_id=None): """ Returns BoxList from `boxes` and adds probability scores information as an extra field `boxes` has shape (#detections, 4 * #classes), where each row represents a list of predicted bounding boxes for each of the object classes in the dataset (including the background class). The detections in each row originate from the same object proposal. `scores` has shape (#detection, #classes), where each row represents a list of object detection confidence scores for each of the object classes in the dataset (including the background class). `scores[i, j]`` corresponds to the box at `boxes[i, j * 4:(j + 1) * 4]`. """ boxes = boxes.reshape(-1, 4) # n*num_cls, 4 scores = scores.reshape(-1) # n*num_cls boxlist = BoxList(boxes, image_shape, mode="xyxy") if cyclic: labels = torch.zeros_like(scores).long().fill_(target_id) boxlist.add_field("labels", labels) boxlist.add_field("scores", scores) return boxlist
def __getitem__(self, idx): # load the image as a PIL Image image = ... # load the bounding boxes as a list of list of boxes # in this case, for illustrative purposes, we use # x1, y1, x2, y2 order. boxes = [[0, 0, 10, 10], [10, 20, 50, 50]] # and labels labels = torch.tensor([10, 20]) # create a BoxList from the boxes boxlist = BoxList(boxes, image.size, mode="xyxy") # add the labels to the boxlist boxlist.add_field("labels", labels) if self.transforms: image, boxlist = self.transforms(image, boxlist) # return the image, the boxlist and the idx in your dataset return image, boxlist, idx
def get_groundtruth(self, ann_info, width, height): # anno = self._preprocess_annotation(self.ann_info[filename], width, height) x1, y1, x2, y2, cls = ann_info x1 = int(x1) y1 = int(y1) x2 = int(x2) y2 = int(y2) cls = int(cls) box = [x1, y1, x2, y2] target = BoxList(torch.tensor([box]), (width, height), mode="xyxy") target.add_field("labels", torch.tensor([cls])) # masks = SegmentationMask(anno["masks"], (width, height)) # masks = SegmentationMask(anno["masks"], (width, height), type=self.mask_type) # target.add_field("masks", masks) return target
def testRegionClassifier(self, model, test_boxes): print('Online Region Classifier testing') predictions = [] total_testing_time = 0 try: for c in range(0, self.num_classes - 1): model[c].ny_points_ = model[c].ny_points_.to('cuda') model[c].alpha_ = model[c].alpha_.to('cuda') except: pass # Convert stats to gpu tensors for inference self.mean = self.mean.to('cuda') self.std = self.std.to('cuda') self.mean_norm = self.mean_norm.to('cuda') for i in range(len(test_boxes)): l = test_boxes[i] if l is not None: I = np.nonzero(l['gt'] == 0) boxes = l['boxes'][I, :][0] X_test = torch.tensor(l['feat'][I, :][0], device='cuda') t0 = time.time() if self.mean_norm != 0: X_test = self.zScores(X_test) scores = -torch.ones((len(boxes), self.num_classes)) for c in range(0, self.num_classes - 1): pred = self.classifier.predict(model[c], X_test) scores[:, c + 1] = torch.squeeze(pred) total_testing_time = total_testing_time + time.time() - t0 b = BoxList(torch.from_numpy(boxes), (l['img_size'][0], l['img_size'][1]), mode="xyxy") b.add_field("scores", scores.to('cpu')) predictions.append(b) avg_time = total_testing_time / len(test_boxes) print('Average image testing time: {} seconds.'.format(avg_time)) return predictions
def get_groundtruth(self, index, evaluation=False, flip_img=False): img_info = self.get_img_info(index) w, h = img_info['width'], img_info['height'] # important: recover original box from BOX_SCALE box = self.gt_boxes[index] / BOX_SCALE * max(w, h) box = torch.from_numpy(box).reshape(-1, 4) # guard against no boxes if flip_img: new_xmin = w - box[:, 2] new_xmax = w - box[:, 0] box[:, 0] = new_xmin box[:, 2] = new_xmax target = BoxList(box, (w, h), 'xyxy') # xyxy target.add_field("labels", torch.from_numpy(self.gt_classes[index])) target.add_field("attributes", torch.from_numpy(self.gt_attributes[index])) relation = self.relationships[index].copy() # (num_rel, 3) if self.filter_duplicate_rels: # Filter out dupes! assert self.split == 'train' old_size = relation.shape[0] all_rel_sets = defaultdict(list) for (o0, o1, r) in relation: all_rel_sets[(o0, o1)].append(r) relation = [(k[0], k[1], np.random.choice(v)) for k, v in all_rel_sets.items()] relation = np.array(relation, dtype=np.int32) # add relation to target num_box = len(target) relation_map = torch.zeros((num_box, num_box), dtype=torch.int64) for i in range(relation.shape[0]): if relation_map[int(relation[i, 0]), int(relation[i, 1])] > 0: if (random.random() > 0.5): relation_map[int(relation[i, 0]), int(relation[i, 1])] = int(relation[i, 2]) else: relation_map[int(relation[i, 0]), int(relation[i, 1])] = int(relation[i, 2]) target.add_field("relation", relation_map, is_triplet=True) if evaluation: target = target.clip_to_image(remove_empty=False) target.add_field("relation_tuple", torch.LongTensor(relation)) # for evaluation return target else: target = target.clip_to_image(remove_empty=True) return target
def get_groundtruth(self, index, call=False): #print ('yes2') row = self.tsvfile.seek(index) anno = json.loads(row[1]) image_data = row[2] base64_data = base64.b64decode(image_data) base64_data = StringIO(base64_data) img = Image.open(base64_data).convert("RGB") anno["boxes"] = torch.tensor(anno["boxes"], dtype=torch.float32) anno["labels"] = torch.tensor(anno["labels"]) anno["attributes"] = torch.tensor(anno["attributes"]) anno["relations"] = torch.tensor(anno["relations"]) height, width = anno["im_info"] target = BoxList(anno["boxes"], (width, height), mode="xyxy") target.add_field("labels", anno["labels"]) target.add_field("attributes", anno["attributes"]) target.add_field("relations", anno["relations"]) #print ('yes3') if call: return img, target, anno else: return target
def __getitem__(self, idx): import pdb;pdb.set_trace() print("you have reached micr datatset get method") img, anno = super(MICRDataset, self).__getitem__(idx) # filter crowd annotations # TODO might be better to add an extra field anno = [obj for obj in anno if obj["iscrowd"] == 0] boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes target = BoxList(boxes, img.size, mode="xywh").convert("xyxy") classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) if anno and "segmentation" in anno[0]: masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size, mode='poly') target.add_field("masks", masks) if anno and "keypoints" in anno[0]: keypoints = [obj["keypoints"] for obj in anno] keypoints = PersonKeypoints(keypoints, img.size) target.add_field("keypoints", keypoints) target = target.clip_to_image(remove_empty=True) if self._transforms is not None: img, target = self._transforms(img, target) return img, target, idx
def _get_target(self, img, index): # a list of label (x1, y1, x2, y2, class_id, instance_id) labels = self._labels[index] if len(labels) == 0: assert self._include_bg is True, "The image does not has ground truth" bbox = torch.as_tensor(labels).reshape(-1, 4) class_ids = torch.as_tensor(labels) instance_ids = torch.as_tensor(labels) empty_boxlist = BoxList(bbox, img.size, mode="xyxy") empty_boxlist.add_field("labels", class_ids) empty_boxlist.add_field("ids", instance_ids) return empty_boxlist labels = torch.as_tensor(labels).reshape(-1, 6) boxes = labels[:, :4] target = BoxList(boxes, img.size, mode="xyxy") class_ids = labels[:, 4].clone().to(torch.int64) target.add_field("labels", class_ids) instance_ids = labels[:, -1].clone().to(torch.int64) target.add_field("ids", instance_ids) if not self._amodal: target = target.clip_to_image(remove_empty=True) return target