def forward(self, rois, roidb, im_info):
context = torch.device('cuda', rois.get_device())
assert self.batch_rois == -1 or self.batch_rois % self.batch_images == 0, \
'batchimages {} must devide batch_rois {}'.format(self.batch_images, self.batch_rois)
all_rois = rois.data.cpu().numpy()
add_proposals([roidb], all_rois, im_info[:, 2], crowd_thresh=0)
blobs = defaultdict(list)
for im_i, entry in enumerate([roidb]):
frcn_blobs = sample_rois(entry, im_info[im_i, 2], im_i)
for k, v in frcn_blobs.items():
blobs[k].append(v)
return torch.cat([torch.tensor(_, dtype=torch.float32, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['rois']], 0),\
torch.cat([torch.tensor(_, dtype=torch.int64, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['labels_int32']], 0),\
torch.cat([torch.tensor(_, dtype=torch.float32, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['bbox_targets']], 0),\
torch.cat([torch.tensor(_, dtype=torch.float32, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['bbox_inside_weights']], 0),\
torch.cat([torch.tensor(_, dtype=torch.float32, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['bbox_outside_weights']], 0),\
torch.cat([torch.tensor(_, dtype=torch.float32, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['mask_rois']], 0),\
torch.cat([torch.tensor(_, dtype=torch.float32, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['mask_int32']], 0), \
torch.cat([torch.tensor(_, dtype=torch.uint8, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['roi_has_mask_int32']], 0), \
torch.cat([torch.tensor(_, dtype=torch.int64, requires_grad=False).pin_memory().to(context, non_blocking=True) for _ in blobs['nongt_inds']], 0)
def forward(self, rois, gt_boxes, gt_masks):
context = torch.device('cuda', rois.get_device())
assert self.batch_rois == -1 or self.batch_rois % self.batch_images == 0, \
'batchimages {} must devide batch_rois {}'.format(self.batch_images, self.batch_rois)
all_rois = rois.cpu().numpy()
gt_boxes = gt_boxes.cpu().numpy()
gt_masks = gt_masks.cpu().numpy()
if self.batch_rois == -1:
rois_per_image = all_rois.shape[0] + gt_boxes.shape[0]
fg_rois_per_image = rois_per_image
else:
rois_per_image = self.batch_rois // self.batch_images
fg_rois_per_image = np.round(self.fg_fraction *
rois_per_image).astype(int)
# Include ground-truth boxes in the set of candidate rois
zeros = np.zeros((gt_boxes.shape[0], 1), dtype=gt_boxes.dtype)
all_rois = np.vstack((all_rois, np.hstack((zeros, gt_boxes[:, :-1]))))
# Sanity check: single batch only
assert np.all(
all_rois[:, 0] == 0), 'Only single item batches are supported'
rois, labels, bbox_targets, bbox_weights, mask_targets, mask_weights = \
sample_rois(all_rois, fg_rois_per_image, rois_per_image, self.num_classes, gt_boxes=gt_boxes,
gt_masks=gt_masks, mask_size=self.mask_size, binary_thresh=self.binary_thresh)
return torch.Tensor(rois).pin_memory().to(context, dtype=torch.float32, non_blocking=True), torch.Tensor(labels).pin_memory().to(context, dtype=torch.int64, non_blocking=True),\
torch.Tensor(bbox_targets).pin_memory().to(context, dtype=torch.float32, non_blocking=True), torch.Tensor(bbox_weights).pin_memory().to(context, dtype=torch.float32, non_blocking=True),\
torch.Tensor(mask_targets).pin_memory().to(context, dtype=torch.float32, non_blocking=True), torch.Tensor(mask_weights).pin_memory().to(context, dtype=torch.float32, non_blocking=True)
def forward(self, rois, gt_boxes):
assert self.batch_rois == -1 or self.batch_rois % self.batch_images == 0, \
'batchimages {} must devide batch_rois {}'.format(self.batch_images, self.batch_rois)
all_rois = rois.cpu().numpy()
gt_boxes = gt_boxes.cpu().numpy()
if self.batch_rois == -1:
rois_per_image = all_rois.shape[0] + gt_boxes.shape[0]
fg_rois_per_image = rois_per_image
else:
rois_per_image = self.batch_rois // self.batch_images
fg_rois_per_image = np.round(self.fg_fraction * rois_per_image).astype(int)
# Include ground-truth boxes in the set of candidate rois
zeros = np.zeros((gt_boxes.shape[0], 1), dtype=gt_boxes.dtype)
all_rois = np.vstack((all_rois, np.hstack((zeros, gt_boxes[:, :-1]))))
# Sanity check: single batch only
assert np.all(all_rois[:, 0] == 0), 'Only single item batches are supported'
rois, labels, bbox_targets, bbox_weights = \
sample_rois(all_rois, fg_rois_per_image, rois_per_image, self.num_classes, gt_boxes=gt_boxes)
return torch.FloatTensor(rois), torch.FloatTensor(labels), torch.FloatTensor(bbox_targets), torch.FloatTensor(bbox_weights)
def __getitem__(self, index):
blob = defaultdict(list)
im_blob, im_scales = self.get_image_blob([self.roidb[index]])
if config.network.has_rpn:
if self.phase != 'test':
add_rpn_blobs(blob, im_scales, [self.roidb[index]])
data = {'data': im_blob, 'im_info': blob['im_info']}
label = {'roidb': blob['roidb'][0]}
for stride in config.network.rpn_feat_stride:
label.update({
'rpn_labels_fpn{}'.format(stride):
blob['rpn_labels_int32_wide_fpn{}'.format(
stride)].astype(np.int64),
'rpn_bbox_targets_fpn{}'.format(stride):
blob['rpn_bbox_targets_wide_fpn{}'.format(stride)],
'rpn_bbox_inside_weights_fpn{}'.format(stride):
blob['rpn_bbox_inside_weights_wide_fpn{}'.format(
stride)],
'rpn_bbox_outside_weights_fpn{}'.format(stride):
blob['rpn_bbox_outside_weights_wide_fpn{}'.format(
stride)]
})
else:
data = {
'data':
im_blob,
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32),
}
label = {'roidb': self.roidb[index]}
else:
if self.phase != 'test':
frcn_blob = sample_rois(self.roidb[index], im_scales, 0)
data = {
'data':
im_blob,
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32)
}
label = {
'rois':
frcn_blob['rois'].astype(np.float32),
'cls_label':
frcn_blob['labels_int32'].astype(np.int64),
'bbox_target':
frcn_blob['bbox_targets'].astype(np.float32),
'bbox_inside_weight':
frcn_blob['bbox_inside_weights'].astype(np.float32),
'bbox_outside_weight':
frcn_blob['bbox_outside_weights'].astype(np.float32),
'mask_rois':
frcn_blob['mask_rois'].astype(np.float32),
'mask_target':
frcn_blob['mask_int32'].astype(np.float32)
}
else:
data = {
'data':
im_blob,
'rois':
np.hstack((np.zeros(
(self.roidb[index]['boxes'].shape[0], 1)),
self.roidb[index]['boxes'])).astype(np.float32),
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32),
'id':
self.roidb[index]['id']
}
label = None
if config.network.has_fcn_head:
if self.phase != 'test':
seg_gt = np.array(
Image.open(self.roidb[index]['image'].replace(
'images',
'labels').replace('leftImg8bit.png',
'gtFine_labelTrainIds.png')))
if self.roidb[index]['flipped']:
seg_gt = np.fliplr(seg_gt)
seg_gt = cv2.resize(seg_gt,
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
label.update({'seg_gt': seg_gt})
label.update({'gt_classes': label['roidb']['gt_classes']})
label.update({
'mask_gt':
np.zeros((len(label['gt_classes']), im_blob.shape[-2],
im_blob.shape[-1]))
})
for i in range(len(label['gt_classes'])):
img = Image.new('L',
(int(im_blob.shape[-1] / im_scales[0]),
int(im_blob.shape[-2] / im_scales[0])), 0)
for j in range(len(label['roidb']['segms'][i])):
ImageDraw.Draw(img).polygon(tuple(
label['roidb']['segms'][i][j]),
outline=1,
fill=1)
label['mask_gt'][i] = cv2.resize(
np.array(img),
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
if config.train.fcn_with_roi_loss:
gt_boxes = label['roidb']['boxes'][np.where(
label['roidb']['gt_classes'] > 0)[0]]
gt_boxes = np.around(gt_boxes * im_scales[0]).astype(
np.int32)
label.update({
'seg_roi_gt':
np.zeros((len(gt_boxes), config.network.mask_size,
config.network.mask_size),
dtype=np.int64)
})
for i in range(len(gt_boxes)):
if gt_boxes[i][3] == gt_boxes[i][1]:
gt_boxes[i][3] += 1
if gt_boxes[i][2] == gt_boxes[i][0]:
gt_boxes[i][2] += 1
label['seg_roi_gt'][i] = cv2.resize(
seg_gt[gt_boxes[i][1]:gt_boxes[i][3],
gt_boxes[i][0]:gt_boxes[i][2]],
(config.network.mask_size,
config.network.mask_size),
interpolation=cv2.INTER_NEAREST)
else:
pass
return data, label, index