def __getitem__(self, idx):
img, anno = super(COCODataset, self).__getitem__(idx)
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size)
target.add_field("masks", masks)
target = target.clip_to_image(remove_empty=True)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, idx
def select_over_all_levels(self, boxlists):
num_images = len(boxlists)
results = []
for i in range(num_images):
scores = boxlists[i].get_field("scores")
labels = boxlists[i].get_field("labels")
boxes = boxlists[i].bbox
boxlist = boxlists[i]
result = []
# skip the background
for j in range(1, self.num_classes):
inds = (labels == j).nonzero().view(-1)
scores_j = scores[inds]
boxes_j = boxes[inds, :].view(-1, 4)
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class = boxlist_nms(boxlist_for_class,
self.nms_thresh,
score_field="scores")
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels",
torch.full((num_labels, ),
j,
dtype=torch.int64,
device=scores.device))
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > self.fpn_post_nms_top_n > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(),
number_of_detections - self.fpn_post_nms_top_n + 1)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
results.append(result)
return results
def __getitem__(self, index):
img_id = self.ids[index]
im_path = os.path.join(self.root, img_id + '.jpg')
img = Image.open(im_path).convert("RGB")
im = cv2.imread(im_path)
anno = self.get_groundtruth(index)
anno["im_info"] = [im.shape[0], im.shape[1]]
height, width = anno["im_info"]
target = BoxList(anno["boxes"], (width, height), mode="xyxy")
target.add_field("labels", anno["labels"])
target.add_field("difficult", anno["difficult"])
target = target.clip_to_image(remove_empty=True)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, index
def remove_low_score_boxes(self, boxlist, num_classes):
boxes = boxlist.bbox
scores = boxlist.get_field("scores")
device = scores.device
num_boxes = int(boxlist.bbox.shape[0] / num_classes)
box_labels = torch.arange(num_classes).repeat(num_boxes).to(device)
keep_inds = (box_labels > 0) * (scores > self.score_thresh)
boxes = boxes[keep_inds, :]
box_labels = box_labels[keep_inds]
scores = scores[keep_inds]
boxelist_filtered = BoxList(boxes, boxlist.size, "xyxy")
boxelist_filtered.add_field("scores", scores)
boxelist_filtered.add_field("box_labels", box_labels)
return boxelist_filtered
def forward(self, x, boxes):
"""
Arguments:
x (Tensor): the mask logits
boxes (list[BoxList]): bounding boxes that are used as
reference, one for ech image
Returns:
results (list[BoxList]): one BoxList for each image, containing
the extra field mask
"""
mask_prob = x.sigmoid()
# select masks coresponding to the predicted classes
num_masks = x.shape[0]
labels = [bbox.get_field("labels") for bbox in boxes] # the id labels
labels = torch.cat(labels)
index = torch.arange(num_masks,
device=labels.device) # the predict boxes
mask_prob = mask_prob[
index, labels][:, None] # select the mask of the box with label
boxes_per_image = [len(box) for box in boxes]
mask_prob = mask_prob.split(boxes_per_image, dim=0)
if self.masker:
# height, width = images.tensors.shape[-2:]
# boxes_init = []
# for i in boxes:
# boxes_init.append(i.resize((width, height)))
# mask_prob = self.masker(mask_prob, boxes_init)
mask_prob = self.masker(mask_prob, boxes)
results = []
for prob, box in zip(mask_prob, boxes):
bbox = BoxList(box.bbox, box.size, mode="xyxy")
for field in box.fields():
bbox.add_field(field, box.get_field(field))
bbox.add_field("mask", prob)
results.append(bbox)
return results
def prepare_boxlist(boxes, scores, image_shape, ids):
"""
Returns BoxList from `boxes` and adds probability scores information
as an extra field
`boxes` has shape (#detections, 4 * #classes), where each row represents
a list of predicted bounding boxes for each of the object classes in the
dataset (including the background class). The detections in each row
originate from the same object proposal.
`scores` has shape (#detection, #classes), where each row represents a list
of object detection confidence scores for each of the object classes in the
dataset (including the background class). `scores[i, j]`` corresponds to the
box at `boxes[i, j * 4:(j + 1) * 4]`.
"""
boxes = boxes.reshape(-1, 4)
scores = scores.reshape(-1)
boxlist = BoxList(boxes, image_shape, mode="xyxy")
boxlist.add_field("scores", scores)
boxlist.add_field("ids", ids)
return boxlist
def filter_results(self, boxlist, num_classes):
"""Returns bounding-box detection results by thresholding on scores and
applying non-maximum suppression (NMS).
"""
# unwrap the boxlist to avoid additional overhead.
# if we had multi-class NMS, we could perform this directly on the boxlist
boxes = boxlist.bbox.reshape(-1, num_classes * 4)
scores = boxlist.get_field("scores").reshape(-1, num_classes)
device = scores.device
result = []
# Apply threshold on detection probabilities and apply NMS
# Skip j = 0, because it's the background class
inds_all = scores > self.score_thresh
for j in range(1, num_classes):
inds = inds_all[:, j].nonzero().squeeze(1)
scores_j = scores[inds, j]
boxes_j = boxes[inds, j * 4:(j + 1) * 4]
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class = boxlist_nms(boxlist_for_class,
self.nms,
score_field="scores")
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels",
torch.full((num_labels, ), j, dtype=torch.int64,
device=device))
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > self.detections_per_img > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(),
number_of_detections - self.detections_per_img + 1)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
return result
def __call__(self, image, target=None):
if target == None:
return image
if not self.do:
return image, target
w, h = image.size
assert w == h
cx = w / 2
cy = cx
degree = random.uniform(0, 360)
radian = degree * math.pi / 180
new_image = image.rotate(-degree)
sin = math.sin(radian)
cos = math.cos(radian)
masks = target.get_field("masks")
polygons = list(map(lambda x: x.polygons[0], masks.instances.polygons))
polygons = torch.stack(polygons, 0).reshape((-1, 2)).t()
M = torch.Tensor([[cos, -sin], [sin, cos]])
b = torch.Tensor([[(1 - cos) * cx + cy * sin],
[(1 - cos) * cy - cx * sin]])
new_points = M.mm(polygons) + b
new_points = new_points.t().reshape((-1, 8))
xmins, _ = torch.min(new_points[:, ::2], 1)
ymins, _ = torch.min(new_points[:, 1::2], 1)
xmaxs, _ = torch.max(new_points[:, ::2], 1)
ymaxs, _ = torch.max(new_points[:, 1::2], 1)
boxes = torch.stack([xmins, ymins, xmaxs, ymaxs], 1).reshape((-1, 4))
new_target = BoxList(boxes, image.size, mode="xyxy")
new_target._copy_extra_fields(target)
new_masks = SegmentationMask(new_points.reshape((-1, 1, 8)).tolist(),
image.size,
mode='poly')
new_target.add_field("masks", new_masks)
return new_image, new_target
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: Tuple (image, target). target is a list of captions for the image.
"""
img_id, sent_id = self.ids[index].split(
'\t')[0], self.ids[index].split('\t')[1]
topN_box = self.topN_box_anno[img_id][int(sent_id)]
filename = os.path.join(self.img_root, img_id + '.jpg')
img = Image.open(filename).convert('RGB')
sent_sg = self.sg_anno[img_id]['relations'][int(sent_id)]
_, feature_map, precompute_bbox, img_scale, precompute_score, cls_label = self.get_precompute_img_feat(
img_id)
precompute_bbox = BoxList(precompute_bbox, img.size, mode='xyxy')
if cfg.MODEL.VG.USE_BOTTOMUP_NMS:
precompute_bbox.add_field("scores",
torch.FloatTensor(precompute_score))
precompute_bbox, keep_inds = boxlist_nms(
precompute_bbox,
cfg.MODEL.VG.BOTTOMUP_NMS_THRESH,
require_keep_idx=True)
precompute_score = precompute_score[keep_inds.numpy()]
sentence = self.get_sentence(img_id, int(sent_id))
phrase_ids, gt_boxes = self.get_gt_boxes(img_id)
target = BoxList(gt_boxes, img.size, mode="xyxy")
vocab_label_elmo = self.vocab_embed[cls_label]
if self.transforms is not None:
img, target, precompute_bbox, img_scale = self.transforms(
img, target, precompute_bbox, img_scale)
return None, target, img_id, phrase_ids, sent_id, sentence, precompute_bbox, precompute_score, feature_map, vocab_label_elmo, sent_sg, topN_box
def __getitem__(self, idx):
# use zipreader, change the function of super.getitem
coco = self.coco
img_id = self.ids[idx]
ann_ids = coco.getAnnIds(imgIds=img_id)
anno = coco.loadAnns(ann_ids)
path = coco.loadImgs(img_id)[0]['file_name']
img = Image.open(os.path.join(self.root, path)).convert('RGB')
# In philly cluster use zipreader instead Image.open
# img = zipreader.imread(os.path.join(self.root, path), \
# cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
# img = Image.fromarray(img)
# img = cv2.imread(os.path.join(self.root, path), cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
# img = Image.fromarray(img)
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size)
target.add_field("masks", masks)
target = target.clip_to_image(remove_empty=True)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, idx
def __getitem__(self, idx):
if self.mode == 0:
img, anno = super(COCODataset, self).__getitem__(idx)
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size)
target.add_field("masks", masks)
target = target.clip_to_image(remove_empty=True)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, idx
elif self.mode == 1:
img_id = self.ids[index]
feature_list = torch.load(self._split_feature % (self.backbone, self.resolution, img_id)
, map_location=torch.device("cpu"))
if self.special_deal:
label = torch.load(self._split_label % (self.backbone, self.resolution, img_id)
, map_location=torch.device("cpu"))
reg = torch.load(self._split_reg % (self.backbone, self.resolution, img_id)
, map_location=torch.device("cpu"))
return feature_list, label, reg, index
else:
target = torch.load(self._split_target % (self.backbone, self.resolution, img_id)
, map_location=torch.device("cpu"))
return feature_list, target, index
else:
raise ValueError("Mode {} do not support now".format(self.mode))
def convert_given_detections_to_boxlist(entities: [AnnoEntity], video_width, video_height,
class_table=None):
# default class is person only
if class_table is None:
class_table = ["person"]
boxes = [_entity.bbox for _entity in entities]
boxes = torch.as_tensor(boxes).reshape(-1, 4)
_labels = [class_table.index(list(_entity.labels.keys())[0]) + 1 for _entity in entities]
_labels = torch.tensor(_labels, dtype=torch.int64)
_scores = torch.tensor([_entity.confidence for _entity in entities])
_ids = torch.tensor([-1 for _entity in entities], dtype=torch.int64)
boxlist = BoxList(boxes,
[video_width, video_height],
mode='xywh').convert('xyxy')
boxlist.add_field('labels', _labels)
boxlist.add_field('scores', _scores)
boxlist.add_field('ids', _ids)
return boxlist
def json2boxlist(imgs):
results = {}
for img, boxs in imgs.items():
i = Image.open(os.path.join(img_dir, img))
boxes = [obj["bbox"] for obj in boxs]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, i.size, mode="xywh").convert("xyxy")
classes = [obj["category_id"] for obj in boxs]
# classes = [c for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
scores = [obj['score'] for obj in boxs]
scores = torch.tensor(scores)
target.add_field("scores", scores)
results[img] = target
return results
def _get_instance_box_list(self, img, anno, image_id):
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")
# assume these are consistent.
classes = [obj["category_id"] for obj in anno]
classes = [self.json_thing_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size)
target.add_field("masks", masks)
return target
def forward(self, boxes, pred_maskiou, labels):
num_masks = pred_maskiou.shape[0]
index = torch.arange(num_masks, device=labels.device)
maskious = pred_maskiou[index, labels]
maskious = [maskious]
results = []
count = 0
# for maskiou, box in zip(maskious, boxes):
for box in boxes:
bbox = BoxList(box.bbox, box.size, mode="xyxy")
for field in box.fields():
bbox.add_field(field, box.get_field(field))
bbox_scores = bbox.get_field("scores")
mask_scores = bbox_scores * maskious[0][count:count +
len(bbox_scores)]
bbox.add_field("mask_scores", mask_scores)
results.append(bbox)
count += len(bbox_scores)
return results
def __getitem__(self, idx):
image, anno = self.pull_item(idx)
labels = torch.from_numpy(np.array(anno['labels']))
# create a BoxList from the boxes
# image need to be a PIL Image
boxlist = BoxList(anno['bndboxes'], image.size, mode="xyxy")
# add the labels to the boxlist
boxlist.add_field("labels", labels)
# add masks
if anno and "polygons" in anno:
masks = SegmentationMask(anno['polygons'], image.size, mode='poly')
boxlist.add_field("masks", masks)
if self._transforms:
image, boxlist = self._transforms(image, boxlist)
# return the image, the boxlist and the idx in your dataset
return image, boxlist, idx
def get_annotation(self, image_id):
coco = self.coco
ann_ids = coco.getAnnIds(imgIds=image_id)
img_data = self.coco.imgs[image_id]
anno = coco.loadAnns(ann_ids)
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, (img_data['width'], img_data['height']),
mode="xywh").convert("xyxy")
labels = [obj["category_id"] for obj in anno]
labels = [self.json_category_id_to_contiguous_id[c] for c in labels]
target.add_field("labels", torch.tensor(labels))
target = target.clip_to_image(remove_empty=True)
return {
'boxes': target.bbox.tolist(),
'labels': target.get_field('labels').tolist()
}
def __getitem__(self, idx):
img_path = self.frames_list[idx]
self.img = Image.open(open(img_path, 'rb'))
boxes, labels = self.get_groundtruth(img_path)
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, self.img.size, mode="xyxy")
# create a BoxList from the boxes
# add the labels to the boxlist
classes = [self.class_to_ind['tire'] for label in labels]
classes = torch.tensor(classes)
target.add_field("labels", classes)
target = target.clip_to_image(remove_empty=True)
if self.transforms:
self.img, target = self.transforms(self.img, target)
return self.img, target, idx
def __getitem__(self, index):
image_id = self.images[index]['id']
img_path = os.path.join(self.images_dir, self.images[index]['file_name'])
img = Image.open(img_path).convert("RGB")
width, height = img.size[0], img.size[1]
boxes = []
labels = []
ann = self.annotations[image_id]
for category, x, y, w, h in ann:
boxes.append([x, y, x + w, y + h])
labels.append(category)
target = BoxList(torch.tensor(boxes, dtype=torch.float32), (width, height), mode="xyxy")
target.add_field('labels', torch.tensor(labels))
target = target.clip_to_image(remove_empty=True)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, index
def forward_for_single_feature_map(self, anchors, objectness,
box_regression):
"""
Arguments:
anchors: list[BoxList]
objectness: tensor of size N, A, H, W
box_regression: tensor of size N, A * 4, H, W
"""
device = objectness.device
N, A, H, W = objectness.shape
objectness, topk_idx, box_regression = self.objectness_top_k(
objectness, box_regression)
batch_idx = torch.arange(N, device=device)[:, None]
box_regression = box_regression[batch_idx, topk_idx]
image_shapes = [box.size for box in anchors]
concat_anchors = torch.cat([a.bbox for a in anchors], dim=0)
concat_anchors = concat_anchors.reshape(N, -1, 4)[batch_idx, topk_idx]
proposals = self.box_coder.decode(box_regression.view(-1, 4),
concat_anchors.view(-1, 4))
proposals = proposals.view(N, -1, 4)
result = []
for proposal, score, im_shape in zip(proposals, objectness,
image_shapes):
boxlist = BoxList(proposal, im_shape, mode="xyxy")
boxlist.add_field("objectness", score)
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
boxlist = boxlist_nms(
boxlist,
self.nms_thresh,
max_proposals=self.post_nms_top_n,
score_field="objectness",
)
result.append(boxlist)
return result
def get_groundtruth(self, idx):
bboxes = []
masks = []
svg_name = self.data_dir / self.split / "{:s}.svg".format(
self.ids[idx])
with svg_name.open('r') as f_svg:
svg = f_svg.read()
num_paths = svg.count('polyline')
for i in range(1, num_paths + 1):
svg_xml = et.fromstring(svg)
svg_xml[1] = svg_xml[i]
del svg_xml[2:]
svg_one = et.tostring(svg_xml, method='xml')
# leave only one path
y_png = cairosvg.svg2png(bytestring=svg_one)
y_img = Image.open(io.BytesIO(y_png))
mask = (np.array(y_img)[:, :, 3] > 0)
try:
bboxes.append(self.get_bbox(mask, "xyxy"))
except:
continue
masks.append(mask.astype(np.uint8))
# if there is completely no boxes, add dummy boxes and masks
if len(bboxes) == 0:
print(self.ids[idx], et.tostring(svg_xml, method='xml'))
dummy_mask = np.zeros(mask.shape, dtype=np.uint8)
dummy_mask[0:4, 0:4] = 1
masks.append(dummy_mask.astype(np.uint8))
bboxes.append(self.get_bbox(dummy_mask, "xyxy"))
image_size_dict = self.get_img_info(idx)
image_size = (image_size_dict["height"], image_size_dict["width"])
boxlist = BoxList(bboxes, image_size, mode="xyxy")
boxlist.add_field("labels", torch.tensor([1] * len(bboxes)))
boxlist.add_field("mask", SegmentationMask(masks, image_size, "mask"))
return self.svg_to_png(svg), boxlist
def forward(self, x, boxes):
"""
Arguments:
x (Tensor): the mask logits
boxes (list[BoxList]): bounding boxes that are used as
reference, one for each image
Returns:
results (list[BoxList]): one BoxList for each image, containing
the extra field mask
"""
if x.device.type == 'mlu':
mask_prob = x.sigmoid()
bbox = boxes[0]
label = boxes[1]
score = boxes[2]
return (mask_prob, bbox, label, score)
mask_prob = x.sigmoid()
# select masks coresponding to the predicted classes
num_masks = x.shape[0]
labels = [bbox.get_field("labels") for bbox in boxes]
labels = torch.cat(labels)
index = torch.arange(num_masks, device=labels.device)
mask_prob = mask_prob[index, labels][:, None]
boxes_per_image = [len(box) for box in boxes]
mask_prob = mask_prob.split(boxes_per_image, dim=0)
if self.masker:
mask_prob = self.masker(mask_prob, boxes)
results = []
for prob, box in zip(mask_prob, boxes):
bbox = BoxList(box.bbox, box.size, mode="xyxy")
for field in box.fields():
bbox.add_field(field, box.get_field(field))
bbox.add_field("mask", prob)
results.append(bbox)
return results
def build_boxlist(fname, single_block=False):
with open(fname, 'rb') as f:
xml = et.fromstring(f.read())
# image dimensions
width, height = (int(xml.attrib[x]) for x in ['width', 'height'])
def xyxy(elt):
return [float(elt.attrib[x]) for x in 'ltrb']
boxes = [xyxy(elt) for elt in xml.findall('.//block')]
if single_block:
boxes = boxes[:1]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, (width, height), mode="xyxy")
classes = torch.tensor([1]*len(boxes), dtype=torch.int32)
target.add_field('labels', classes)
return target
def prepare_boxlist(self, boxes, scores, image_shape, cyclic=False, target_id=None):
"""
Returns BoxList from `boxes` and adds probability scores information
as an extra field
`boxes` has shape (#detections, 4 * #classes), where each row represents
a list of predicted bounding boxes for each of the object classes in the
dataset (including the background class). The detections in each row
originate from the same object proposal.
`scores` has shape (#detection, #classes), where each row represents a list
of object detection confidence scores for each of the object classes in the
dataset (including the background class). `scores[i, j]`` corresponds to the
box at `boxes[i, j * 4:(j + 1) * 4]`.
"""
boxes = boxes.reshape(-1, 4) # n*num_cls, 4
scores = scores.reshape(-1) # n*num_cls
boxlist = BoxList(boxes, image_shape, mode="xyxy")
if cyclic:
labels = torch.zeros_like(scores).long().fill_(target_id)
boxlist.add_field("labels", labels)
boxlist.add_field("scores", scores)
return boxlist
def __getitem__(self, idx):
# load the image as a PIL Image
image = ...
# load the bounding boxes as a list of list of boxes
# in this case, for illustrative purposes, we use
# x1, y1, x2, y2 order.
boxes = [[0, 0, 10, 10], [10, 20, 50, 50]]
# and labels
labels = torch.tensor([10, 20])
# create a BoxList from the boxes
boxlist = BoxList(boxes, image.size, mode="xyxy")
# add the labels to the boxlist
boxlist.add_field("labels", labels)
if self.transforms:
image, boxlist = self.transforms(image, boxlist)
# return the image, the boxlist and the idx in your dataset
return image, boxlist, idx
def get_groundtruth(self, ann_info, width, height):
# anno = self._preprocess_annotation(self.ann_info[filename], width, height)
x1, y1, x2, y2, cls = ann_info
x1 = int(x1)
y1 = int(y1)
x2 = int(x2)
y2 = int(y2)
cls = int(cls)
box = [x1, y1, x2, y2]
target = BoxList(torch.tensor([box]), (width, height), mode="xyxy")
target.add_field("labels", torch.tensor([cls]))
# masks = SegmentationMask(anno["masks"], (width, height))
# masks = SegmentationMask(anno["masks"], (width, height), type=self.mask_type)
# target.add_field("masks", masks)
return target
def testRegionClassifier(self, model, test_boxes):
print('Online Region Classifier testing')
predictions = []
total_testing_time = 0
try:
for c in range(0, self.num_classes - 1):
model[c].ny_points_ = model[c].ny_points_.to('cuda')
model[c].alpha_ = model[c].alpha_.to('cuda')
except:
pass
# Convert stats to gpu tensors for inference
self.mean = self.mean.to('cuda')
self.std = self.std.to('cuda')
self.mean_norm = self.mean_norm.to('cuda')
for i in range(len(test_boxes)):
l = test_boxes[i]
if l is not None:
I = np.nonzero(l['gt'] == 0)
boxes = l['boxes'][I, :][0]
X_test = torch.tensor(l['feat'][I, :][0], device='cuda')
t0 = time.time()
if self.mean_norm != 0:
X_test = self.zScores(X_test)
scores = -torch.ones((len(boxes), self.num_classes))
for c in range(0, self.num_classes - 1):
pred = self.classifier.predict(model[c], X_test)
scores[:, c + 1] = torch.squeeze(pred)
total_testing_time = total_testing_time + time.time() - t0
b = BoxList(torch.from_numpy(boxes),
(l['img_size'][0], l['img_size'][1]),
mode="xyxy")
b.add_field("scores", scores.to('cpu'))
predictions.append(b)
avg_time = total_testing_time / len(test_boxes)
print('Average image testing time: {} seconds.'.format(avg_time))
return predictions
def get_groundtruth(self, index, evaluation=False, flip_img=False):
img_info = self.get_img_info(index)
w, h = img_info['width'], img_info['height']
# important: recover original box from BOX_SCALE
box = self.gt_boxes[index] / BOX_SCALE * max(w, h)
box = torch.from_numpy(box).reshape(-1, 4) # guard against no boxes
if flip_img:
new_xmin = w - box[:, 2]
new_xmax = w - box[:, 0]
box[:, 0] = new_xmin
box[:, 2] = new_xmax
target = BoxList(box, (w, h), 'xyxy') # xyxy
target.add_field("labels", torch.from_numpy(self.gt_classes[index]))
target.add_field("attributes",
torch.from_numpy(self.gt_attributes[index]))
relation = self.relationships[index].copy() # (num_rel, 3)
if self.filter_duplicate_rels:
# Filter out dupes!
assert self.split == 'train'
old_size = relation.shape[0]
all_rel_sets = defaultdict(list)
for (o0, o1, r) in relation:
all_rel_sets[(o0, o1)].append(r)
relation = [(k[0], k[1], np.random.choice(v))
for k, v in all_rel_sets.items()]
relation = np.array(relation, dtype=np.int32)
# add relation to target
num_box = len(target)
relation_map = torch.zeros((num_box, num_box), dtype=torch.int64)
for i in range(relation.shape[0]):
if relation_map[int(relation[i, 0]), int(relation[i, 1])] > 0:
if (random.random() > 0.5):
relation_map[int(relation[i, 0]),
int(relation[i, 1])] = int(relation[i, 2])
else:
relation_map[int(relation[i, 0]),
int(relation[i, 1])] = int(relation[i, 2])
target.add_field("relation", relation_map, is_triplet=True)
if evaluation:
target = target.clip_to_image(remove_empty=False)
target.add_field("relation_tuple",
torch.LongTensor(relation)) # for evaluation
return target
else:
target = target.clip_to_image(remove_empty=True)
return target
def get_groundtruth(self, index, call=False):
#print ('yes2')
row = self.tsvfile.seek(index)
anno = json.loads(row[1])
image_data = row[2]
base64_data = base64.b64decode(image_data)
base64_data = StringIO(base64_data)
img = Image.open(base64_data).convert("RGB")
anno["boxes"] = torch.tensor(anno["boxes"], dtype=torch.float32)
anno["labels"] = torch.tensor(anno["labels"])
anno["attributes"] = torch.tensor(anno["attributes"])
anno["relations"] = torch.tensor(anno["relations"])
height, width = anno["im_info"]
target = BoxList(anno["boxes"], (width, height), mode="xyxy")
target.add_field("labels", anno["labels"])
target.add_field("attributes", anno["attributes"])
target.add_field("relations", anno["relations"])
#print ('yes3')
if call:
return img, target, anno
else:
return target
def __getitem__(self, idx):
import pdb;pdb.set_trace()
print("you have reached micr datatset get method")
img, anno = super(MICRDataset, self).__getitem__(idx)
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
if anno and "segmentation" in anno[0]:
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size, mode='poly')
target.add_field("masks", masks)
if anno and "keypoints" in anno[0]:
keypoints = [obj["keypoints"] for obj in anno]
keypoints = PersonKeypoints(keypoints, img.size)
target.add_field("keypoints", keypoints)
target = target.clip_to_image(remove_empty=True)
if self._transforms is not None:
img, target = self._transforms(img, target)
return img, target, idx
def _get_target(self, img, index):
# a list of label (x1, y1, x2, y2, class_id, instance_id)
labels = self._labels[index]
if len(labels) == 0:
assert self._include_bg is True, "The image does not has ground truth"
bbox = torch.as_tensor(labels).reshape(-1, 4)
class_ids = torch.as_tensor(labels)
instance_ids = torch.as_tensor(labels)
empty_boxlist = BoxList(bbox, img.size, mode="xyxy")
empty_boxlist.add_field("labels", class_ids)
empty_boxlist.add_field("ids", instance_ids)
return empty_boxlist
labels = torch.as_tensor(labels).reshape(-1, 6)
boxes = labels[:, :4]
target = BoxList(boxes, img.size, mode="xyxy")
class_ids = labels[:, 4].clone().to(torch.int64)
target.add_field("labels", class_ids)
instance_ids = labels[:, -1].clone().to(torch.int64)
target.add_field("ids", instance_ids)
if not self._amodal:
target = target.clip_to_image(remove_empty=True)
return target
