def __getitem__(self, index):
blob = defaultdict(list)
im_blob, im_scales = self.get_image_blob([self.roidb[index]])
if config.network.has_rpn:
if self.phase != 'test':
add_rpn_blobs(blob, im_scales, [self.roidb[index]])
data = {'data': im_blob, 'im_info': blob['im_info']}
label = {'roidb': blob['roidb'][0]}
for stride in config.network.rpn_feat_stride:
label.update({
'rpn_labels_fpn{}'.format(stride):
blob['rpn_labels_int32_wide_fpn{}'.format(
stride)].astype(np.int64),
'rpn_bbox_targets_fpn{}'.format(stride):
blob['rpn_bbox_targets_wide_fpn{}'.format(stride)],
'rpn_bbox_inside_weights_fpn{}'.format(stride):
blob['rpn_bbox_inside_weights_wide_fpn{}'.format(
stride)],
'rpn_bbox_outside_weights_fpn{}'.format(stride):
blob['rpn_bbox_outside_weights_wide_fpn{}'.format(
stride)]
})
else:
data = {
'data':
im_blob,
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32)
}
label = None
else:
raise NotImplementedError
if config.network.has_fcn_head:
if self.phase != 'test':
seg_gt = np.array(
Image.open(self.roidb[index]['image'].replace(
'images', 'annotations'
).replace(
'train2017',
'panoptic_train2017_semantic_trainid_stff').replace(
'val2017',
'panoptic_val2017_semantic_trainid_stff').replace(
'jpg', 'png')))
if self.roidb[index]['flipped']:
seg_gt = np.fliplr(seg_gt)
seg_gt = cv2.resize(seg_gt,
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
label.update({'seg_gt': seg_gt})
label.update({
'gt_classes':
label['roidb']['gt_classes'][label['roidb']['is_crowd'] ==
0]
})
label.update({
'mask_gt':
np.zeros((len(label['gt_classes']), im_blob.shape[-2],
im_blob.shape[-1]))
})
idx = 0
for i in range(len(label['roidb']['gt_classes'])):
if label['roidb']['is_crowd'][i] != 0:
continue
if type(label['roidb']['segms'][i]) is list and type(
label['roidb']['segms'][i][0]) is list:
img = Image.new(
'L',
(int(np.round(im_blob.shape[-1] / im_scales[0])),
int(np.round(im_blob.shape[-2] / im_scales[0]))),
0)
for j in range(len(label['roidb']['segms'][i])):
ImageDraw.Draw(img).polygon(tuple(
label['roidb']['segms'][i][j]),
outline=1,
fill=1)
label['mask_gt'][idx] = cv2.resize(
np.array(img),
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
else:
assert type(
label['roidb']['segms'][i]) is dict or type(
label['roidb']['segms'][i][0]) is dict
if type(label['roidb']['segms'][i]) is dict:
label['mask_gt'][idx] = cv2.resize(
mask_util.decode(
mask_util.frPyObjects(
[label['roidb']['segms'][i]],
label['roidb']['segms'][i]['size'][0],
label['roidb']['segms'][i]['size'][1]))
[:, :, 0],
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
else:
assert len(label['roidb']['segms'][i]) == 1
output = mask_util.decode(
label['roidb']['segms'][i])
label['mask_gt'][idx] = cv2.resize(
output[:, :, 0],
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
idx += 1
if config.train.fcn_with_roi_loss:
gt_boxes = label['roidb']['boxes'][
np.where((label['roidb']['gt_classes'] > 0)
& (label['roidb']['is_crowd'] == 0))[0]]
gt_boxes = np.around(gt_boxes * im_scales[0]).astype(
np.int32)
label.update({
'seg_roi_gt':
np.zeros((len(gt_boxes), config.network.mask_size,
config.network.mask_size),
dtype=np.int64)
})
for i in range(len(gt_boxes)):
if gt_boxes[i][3] == gt_boxes[i][1]:
gt_boxes[i][3] += 1
if gt_boxes[i][2] == gt_boxes[i][0]:
gt_boxes[i][2] += 1
label['seg_roi_gt'][i] = cv2.resize(
seg_gt[gt_boxes[i][1]:gt_boxes[i][3],
gt_boxes[i][0]:gt_boxes[i][2]],
(config.network.mask_size,
config.network.mask_size),
interpolation=cv2.INTER_NEAREST)
else:
pass
return data, label, index
def __getitem__(self, index):
blob = defaultdict(list)
im_blob, im_scales = self.get_image_blob([self.roidb[index]])
if config.network.has_rpn:
if self.phase != 'test':
add_rpn_blobs(blob, im_scales, [self.roidb[index]])
data = {'data': im_blob, 'im_info': blob['im_info']}
label = {'roidb': blob['roidb'][0]}
for stride in config.network.rpn_feat_stride:
label.update({
'rpn_labels_fpn{}'.format(stride):
blob['rpn_labels_int32_wide_fpn{}'.format(
stride)].astype(np.int64),
'rpn_bbox_targets_fpn{}'.format(stride):
blob['rpn_bbox_targets_wide_fpn{}'.format(stride)],
'rpn_bbox_inside_weights_fpn{}'.format(stride):
blob['rpn_bbox_inside_weights_wide_fpn{}'.format(
stride)],
'rpn_bbox_outside_weights_fpn{}'.format(stride):
blob['rpn_bbox_outside_weights_wide_fpn{}'.format(
stride)]
})
else:
data = {
'data':
im_blob,
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32),
}
label = {'roidb': self.roidb[index]}
else:
if self.phase != 'test':
frcn_blob = sample_rois(self.roidb[index], im_scales, 0)
data = {
'data':
im_blob,
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32)
}
label = {
'rois':
frcn_blob['rois'].astype(np.float32),
'cls_label':
frcn_blob['labels_int32'].astype(np.int64),
'bbox_target':
frcn_blob['bbox_targets'].astype(np.float32),
'bbox_inside_weight':
frcn_blob['bbox_inside_weights'].astype(np.float32),
'bbox_outside_weight':
frcn_blob['bbox_outside_weights'].astype(np.float32),
'mask_rois':
frcn_blob['mask_rois'].astype(np.float32),
'mask_target':
frcn_blob['mask_int32'].astype(np.float32)
}
else:
data = {
'data':
im_blob,
'rois':
np.hstack((np.zeros(
(self.roidb[index]['boxes'].shape[0], 1)),
self.roidb[index]['boxes'])).astype(np.float32),
'im_info':
np.array(
[[im_blob.shape[-2], im_blob.shape[-1], im_scales[0]]],
np.float32),
'id':
self.roidb[index]['id']
}
label = None
if config.network.has_fcn_head:
if self.phase != 'test':
seg_gt = np.array(
Image.open(self.roidb[index]['image'].replace(
'images',
'labels').replace('leftImg8bit.png',
'gtFine_labelTrainIds.png')))
if self.roidb[index]['flipped']:
seg_gt = np.fliplr(seg_gt)
seg_gt = cv2.resize(seg_gt,
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
label.update({'seg_gt': seg_gt})
label.update({'gt_classes': label['roidb']['gt_classes']})
label.update({
'mask_gt':
np.zeros((len(label['gt_classes']), im_blob.shape[-2],
im_blob.shape[-1]))
})
for i in range(len(label['gt_classes'])):
img = Image.new('L',
(int(im_blob.shape[-1] / im_scales[0]),
int(im_blob.shape[-2] / im_scales[0])), 0)
for j in range(len(label['roidb']['segms'][i])):
ImageDraw.Draw(img).polygon(tuple(
label['roidb']['segms'][i][j]),
outline=1,
fill=1)
label['mask_gt'][i] = cv2.resize(
np.array(img),
None,
None,
fx=im_scales[0],
fy=im_scales[0],
interpolation=cv2.INTER_NEAREST)
if config.train.fcn_with_roi_loss:
gt_boxes = label['roidb']['boxes'][np.where(
label['roidb']['gt_classes'] > 0)[0]]
gt_boxes = np.around(gt_boxes * im_scales[0]).astype(
np.int32)
label.update({
'seg_roi_gt':
np.zeros((len(gt_boxes), config.network.mask_size,
config.network.mask_size),
dtype=np.int64)
})
for i in range(len(gt_boxes)):
if gt_boxes[i][3] == gt_boxes[i][1]:
gt_boxes[i][3] += 1
if gt_boxes[i][2] == gt_boxes[i][0]:
gt_boxes[i][2] += 1
label['seg_roi_gt'][i] = cv2.resize(
seg_gt[gt_boxes[i][1]:gt_boxes[i][3],
gt_boxes[i][0]:gt_boxes[i][2]],
(config.network.mask_size,
config.network.mask_size),
interpolation=cv2.INTER_NEAREST)
else:
pass
return data, label, index