def _sample_rois(roidb, im_scale, batch_idx, label_code):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI)
& (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
action_sampled_labels = roidb['max_classes'][keep_inds]
action_sampled_labels[
fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
# code.interact(local=locals())
if 'bbox_targets' not in roidb:
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
gt_boxes = roidb['boxes'][gt_inds, :]
gt_assignments = gt_inds[roidb['box_to_gt_ind_map'][keep_inds]]
bbox_targets = _compute_targets(sampled_boxes,
gt_boxes[gt_assignments, :],
sampled_labels)
bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets)
# code.interact(local=locals())
else:
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :])
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# new version for hierachy training
for idx in range(0, len(sampled_labels)):
label_int = sampled_labels[idx]
if label_int != 0:
sampled_labels[idx] = 1
action_sampled_labels[idx] = 0
multi_label_str = label_code['idx_to_label'][str(label_int)]
multi_label = map(int, list(multi_label_str))
for k in range(0, 14):
if int(multi_label[k + 1]) == 1:
action_sampled_labels[idx] = k + 1
else:
sampled_labels[idx] = 0
action_sampled_labels[idx] = 0
# Base Fast R-CNN blobs
blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False),
multilabels_int32=action_sampled_labels.astype(
np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights)
# previous version for multilabel training
'''
expand_sampled_labels = np.expand_dims(sampled_labels,axis=2)
expand_sampled_labels = np.repeat(expand_sampled_labels,81,axis=1)
for idx in range(0,len(sampled_labels)):
label_int = sampled_labels[idx]
if label_int != 0:
sampled_labels[idx] = 1
multi_label_str = label_code['idx_to_label'][str(label_int)]
multi_label = map(int, list(multi_label_str))
expand_sampled_labels[idx] = multi_label
else:
sampled_labels[idx] = 0
expand_sampled_labels[idx] = [0]*81
# Base Fast R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
multilabels_int32=expand_sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights
)
'''
# Base Fast R-CNN blobs
'''
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights
)
'''
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb,
fg_rois_per_image,
fg_inds, im_scale,
batch_idx)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI)
& (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :])
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb,
fg_rois_per_image,
fg_inds, im_scale,
batch_idx)
# optionally add Domain Adaptive R-CNN blobs
if cfg.TRAIN.DOMAIN_ADAPTATION:
if roidb['is_source']:
blob_dict['dc_label'] = np.expand_dims(np.ones(
blob_dict['labels_int32'].shape,
dtype=blob_dict['labels_int32'].dtype),
axis=1)
blob_dict['label_mask'] = np.full(blob_dict['labels_int32'].shape,
True)
blob_dict['source_labels_int32'] = blob_dict['labels_int32']
blob_dict['source_bbox_targets'] = blob_dict['bbox_targets']
blob_dict['source_bbox_inside_weights'] = blob_dict[
'bbox_inside_weights']
blob_dict['source_bbox_outside_weights'] = blob_dict[
'bbox_outside_weights']
blob_dict['da_label_wide'] = np.ones((1, 1, 200, 400),
dtype=np.int32)
else:
blob_dict['dc_label'] = np.expand_dims(np.zeros(
blob_dict['labels_int32'].shape,
dtype=blob_dict['labels_int32'].dtype),
axis=1)
blob_dict['label_mask'] = np.full(blob_dict['labels_int32'].shape,
False)
blob_dict['da_label_wide'] = np.zeros((1, 1, 200, 400),
dtype=np.int32)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
if cfg.REID.PSE_ON:
img_labels = np.array([0], dtype=np.float32)
attr_img_labels = np.array([0], dtype=np.float32)
weight = np.array([0.0], dtype=np.float32)
attr_weight = np.array([0.0], dtype=np.float32)
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
assert len(gt_inds) <= 2, 'Only one ground truth for image is allowed.'
gt_classes = roidb['gt_classes'][gt_inds].copy()
gt_inds = np.where(roidb['gt_attributions'] > 0)[0]
assert len(gt_inds) <= 2, 'Only one ground truth for image is allowed.'
gt_attributions = roidb['gt_attributions'][gt_inds].copy()
classes_or_attributions = roidb['classes_or_attributions']
for i in range(len(gt_classes)):
if classes_or_attributions[i] == 0:
img_labels[0] = gt_classes[i] - 1
weight[0] = 1.0
elif classes_or_attributions[i] == 1:
attr_img_labels[0] = gt_attributions[i] - 1
attr_weight[0] = cfg.REID.PSE_WEIGHT
else:
img_labels[0] = gt_classes[i] - 1
weight[0] = 1.0
attr_img_labels[0] = gt_attributions[i] - 1
attr_weight[0] = cfg.REID.PSE_WEIGHT
blob_dict = dict(
labels_int32=img_labels.astype(np.int32, copy=False),
attr_labels_int32=attr_img_labels.astype(np.int32, copy=False),
weight=weight.astype(np.float32, copy=False),
attr_weight=attr_weight.astype(np.float32, copy=False),
)
return blob_dict
# Get image label
img_labels_oh = np.zeros((1, cfg.MODEL.NUM_CLASSES - 1), dtype=np.float32)
img_labels = np.zeros((1), dtype=np.float32)
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
assert len(gt_inds) == 1, 'Only one ground truth for image is allowed.'
gt_classes = roidb['gt_classes'][gt_inds].copy()
img_labels_oh[0][gt_classes[0] - 1] = 1
img_labels[0] = gt_classes[0] - 1
blob_dict = dict(
labels_int32=img_labels.astype(np.int32, copy=False),
labels_oh=img_labels_oh.astype(np.float32, copy=False),
)
return blob_dict
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(
fg_inds, size=fg_rois_per_this_image, replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
(max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(
bg_inds, size=bg_rois_per_this_image, replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :])
bbox_outside_weights = np.array(
bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(
blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI)
& (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
gt_boxes = roidb['boxes'][gt_inds, :]
gt_assignments = gt_inds[roidb['box_to_gt_ind_map'][keep_inds]]
# [mapped_gt_boxes, max_overlaps]
mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5))
mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale
mapped_gt_boxes[:, 4] = max_overlaps[keep_inds]
mapped_gt_boxes[fg_rois_per_this_image:, :] = 0
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :])
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights,
mapped_gt_boxes=mapped_gt_boxes)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == 1:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == 1:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb,
fg_rois_per_image,
fg_inds, im_scale,
batch_idx,
cfg.TRAIN.FG_THRESH)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(
fg_inds, size=fg_rois_per_this_image, replace=False
)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where(
(max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
(max_overlaps >= cfg.TRAIN.BG_THRESH_LO)
)[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(
bg_inds, size=bg_rois_per_this_image, replace=False
)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :]
)
bbox_outside_weights = np.array(
bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype
)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones((sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights
)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(
blob_dict, sampled_boxes, roidb, im_scale, batch_idx
)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(
blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx
)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, stage_num):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
# set improving rcnn iou threshold for cascade rcnn
bg_thresh_lo = cfg.TRAIN.BG_THRESH_LO
if stage_num == 1:
fg_thresh = cfg.TRAIN.FG_THRESH # 0.5
bg_thresh_hi = cfg.TRAIN.BG_THRESH_HI # 0.5
elif stage_num == 2:
fg_thresh = 0.6
bg_thresh_hi = 0.6
elif stage_num == 3:
fg_thresh = 0.7
bg_thresh_hi = 0.7
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= fg_thresh)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < bg_thresh_hi)
& (max_overlaps >= bg_thresh_lo))[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
sampled_max_overlaps = roidb['max_overlaps'][keep_inds]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :])
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
if stage_num == 1:
blob_dict = dict(labels_int32_1st=sampled_labels.astype(np.int32,
copy=False),
rois_1st=sampled_rois,
bbox_targets_1st=bbox_targets,
bbox_inside_weights_1st=bbox_inside_weights,
bbox_outside_weights_1st=bbox_outside_weights,
max_overlaps_1st=sampled_max_overlaps)
if stage_num == 2:
blob_dict = dict(labels_int32_2nd=sampled_labels.astype(np.int32,
copy=False),
rois_2nd=sampled_rois,
bbox_targets_2nd=bbox_targets,
bbox_inside_weights_2nd=bbox_inside_weights,
bbox_outside_weights_2nd=bbox_outside_weights,
max_overlaps_2nd=sampled_max_overlaps)
if stage_num == 3:
blob_dict = dict(labels_int32_3rd=sampled_labels.astype(np.int32,
copy=False),
rois_3rd=sampled_rois,
bbox_targets_3rd=bbox_targets,
bbox_inside_weights_3rd=bbox_inside_weights,
bbox_outside_weights_3rd=bbox_outside_weights)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes,
roidb, im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(
blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale,
batch_idx)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, stage):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
fg_thresh = cfg.CASCADE_RCNN.FG_THRESHS[stage - 1]
bg_thresh_hi = cfg.CASCADE_RCNN.BG_THRESHS_HI[stage - 1]
bg_thresh_lo = cfg.CASCADE_RCNN.BG_THRESHS_LO[stage - 1]
max_overlaps = roidb["max_overlaps"]
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= fg_thresh)[0]
fg_rois_per_this_image = fg_inds.size
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < bg_thresh_hi)
& (max_overlaps >= bg_thresh_lo))[0]
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb["max_classes"][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb["boxes"][keep_inds]
gt_inds = np.where(roidb["gt_classes"] > 0)[0]
gt_boxes = roidb["boxes"][gt_inds, :]
gt_assignments = gt_inds[roidb["box_to_gt_ind_map"][keep_inds]]
# [mapped_gt_boxes, max_overlaps]
mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5))
mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale
mapped_gt_boxes[:, 4] = max_overlaps[keep_inds]
mapped_gt_boxes[fg_rois_per_this_image:, :] = 0
if "bbox_targets" not in roidb:
bbox_targets = _compute_targets(sampled_boxes,
gt_boxes[gt_assignments, :],
sampled_labels, stage)
else:
bbox_targets = roidb['bbox_targets'][keep_inds, :]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets)
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Cascade R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights,
mapped_gt_boxes=mapped_gt_boxes,
)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == stage:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == stage:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb,
fg_rois_per_this_image,
fg_inds, im_scale,
batch_idx, fg_thresh)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, stage):
# 随机产生包含前景和背景的正负样本，即rois集合
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
fg_thresh = cfg.CASCADE_RCNN.FG_THRESHS[stage - 1]
bg_thresh_hi = cfg.CASCADE_RCNN.BG_THRESHS_HI[stage - 1]
bg_thresh_lo = cfg.CASCADE_RCNN.BG_THRESHS_LO[stage - 1]
#### 这个变量很关键：定义了每一个roi与所有gt-bboxes所对应的最大的iou ???
max_overlaps = roidb["max_overlaps"]
# Select foreground RoIs as those with >= FG_THRESH overlap
#### 选择最大iou高于fg_thresh的那些rois，作为前景的indexes-->fg_inds
fg_inds = np.where(max_overlaps >= fg_thresh)[0]
#### 一张img所包含的所有fg_rois数量
fg_rois_per_this_image = fg_inds.size
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
#### 选择那些对应的最大iou值介于（bg_thesh_lo, bg_thresh_hi）的那些rois，采样为背景
bg_inds = np.where((max_overlaps < bg_thresh_hi)
& (max_overlaps >= bg_thresh_lo))[0]
# The indices that we're selecting (both fg and bg)
#### keep_inds为我们将要采样的样本的索引列表（包括正，负样本）
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
#### sampled_labels即为roidb中采样的rois所对应的标签值labels
sampled_labels = roidb["max_classes"][keep_inds]
#### 将采样的负样本所对应的标签值都置为0
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
#### sampled_boxes即为roidb中采样的所有正负样本，shape为(, 4)
sampled_boxes = roidb["boxes"][keep_inds]
#### 得到roidb中所有的gt_bbox的index
gt_inds = np.where(roidb["gt_classes"] > 0)[0]
#### 得到gt_bbox构成的集合，shape为(, 4)
gt_boxes = roidb["boxes"][gt_inds, :]
#### 这个有点不太理解了，gt_assignments为所有采样的作为正样本的rois所对应的gt_bboxes集合
gt_assignments = gt_inds[roidb["box_to_gt_ind_map"][keep_inds]]
# [mapped_gt_boxes, max_overlaps]
#### 初始化mapped_gt_boxes变量，保存所有采样的rois所对应的gt_bboxes的坐标（
# 疑问：对于负样本也有gt_bbox??? ）
mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5))
#### mapped_gt_boxes
#### 将原图对应的
mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale
mapped_gt_boxes[:, 4] = max_overlaps[keep_inds]
####对于负样本所对应的行（fg_rois_per_this_image之后的）的数据，全部置为0
mapped_gt_boxes[fg_rois_per_this_image:, :] = 0
if "bbox_targets" not in roidb:
bbox_targets = _compute_targets(sampled_boxes,
gt_boxes[gt_assignments, :],
sampled_labels, stage)
else:
bbox_targets = roidb['bbox_targets'][keep_inds, :]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets)
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Cascade R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights,
mapped_gt_boxes=mapped_gt_boxes,
)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == stage:
mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == stage:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb,
fg_rois_per_this_image,
fg_inds, im_scale,
batch_idx, fg_thresh)
return blob_dict
def _sample_rois(roidb, im_scale, im_crop, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
rois_this_image = np.minimum(rois_per_image, roidb['boxes'].shape[0])
if False:
choice = np.random.choice(roidb['boxes'].shape[0],
rois_this_image,
replace=False)
sampled_boxes = roidb['boxes'][choice, :].copy()
obn_scores = roidb['obn_scores'][choice, :].copy()
sampled_scores = np.add(obn_scores, 1.0)
else:
sampled_boxes = roidb['boxes'][:rois_this_image].copy()
obn_scores = roidb['obn_scores'][:rois_this_image].copy()
sampled_scores = np.add(obn_scores, 1.0)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
# sampled_rois = sampled_boxes * im_scale
sampled_rois = _project_im_rois(sampled_boxes, im_scale, im_crop)
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# gt_inds = np.where((roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0))[0]
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
np.delete(sampled_rois, gt_inds, 0)
np.delete(sampled_scores, gt_inds, 0)
if cfg.WSL.CONTEXT and False:
sampled_boxes = roidb['boxes'][:rois_this_image].copy()
sampled_boxes_inner, sampled_boxes_outer = get_inner_outer_rois(
sampled_boxes, cfg.WSL.CONTEXT_RATIO)
sampled_rois_origin = _project_im_rois(sampled_boxes, im_scale,
im_crop)
sampled_rois_inner = _project_im_rois(sampled_boxes_inner, im_scale,
im_crop)
sampled_rois_outer = _project_im_rois(sampled_boxes_outer, im_scale,
im_crop)
repeated_batch_idx_inner = batch_idx * blob_utils.ones(
(sampled_rois_origin.shape[0], 1))
repeated_batch_idx_outer = batch_idx * blob_utils.ones(
(sampled_rois_origin.shape[0], 1))
sampled_rois_frame = np.hstack(
(repeated_batch_idx, sampled_rois_origin, sampled_rois_inner))
sampled_rois_context = np.hstack(
(repeated_batch_idx, sampled_rois_outer, sampled_rois_origin))
# Delete GT Boxes
np.delete(sampled_rois_frame, gt_inds, 0)
np.delete(sampled_rois_context, gt_inds, 0)
# Get image label
img_labels_oh = np.zeros((1, cfg.MODEL.NUM_CLASSES - 1), dtype=np.float32)
img_labels = np.zeros((1), dtype=np.float32)
# gt_inds = np.where((roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0))[0]
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
assert len(gt_inds) > 0, roidb['gt_classes']
assert len(
gt_inds
) > 0, 'Empty ground truth empty for image is not allowed. Please check.'
gt_classes = roidb['gt_classes'][gt_inds].copy()
num_valid_objs = gt_classes.shape[0]
for o in range(num_valid_objs):
img_labels_oh[0][gt_classes[o] - 1] = 1
img_labels[0] = gt_classes[o] - 1
blob_dict = dict(
labels_int32=img_labels.astype(np.int32, copy=False),
labels_oh=img_labels_oh.astype(np.float32, copy=False),
rois=sampled_rois.astype(np.float32, copy=False),
obn_scores=sampled_scores,
)
if cfg.WSL.CONTEXT and False:
blob_dict['rois_frame'] = sampled_rois_frame.astype(np.float32,
copy=False)
blob_dict['rois_context'] = sampled_rois_context.astype(np.float32,
copy=False)
# Optionally add Mask R-CNN blobs
# if cfg.MODEL.MASK_ON:
# mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb,
# im_scale, batch_idx)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb,
fg_rois_per_image,
fg_inds, im_scale,
batch_idx)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
获得fg和bg roi
"""
# 512
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
# 128
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
# 选择overlaps大于给定值的作为正样本
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
# fg的数量
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
# 随机选择fg_rois_per_this_image个正样本
if fg_inds.size > 0:
fg_inds = npr.choice(
fg_inds, size=fg_rois_per_this_image, replace=False
)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where(
(max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
(max_overlaps >= cfg.TRAIN.BG_THRESH_LO)
)[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
# 选择bg目标
if bg_inds.size > 0:
bg_inds = npr.choice(
bg_inds, size=bg_rois_per_this_image, replace=False
)
# The indices that we're selecting (both fg and bg)
# 合并fg和bg
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
# bg rois的标签置为0
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :]
)
bbox_outside_weights = np.array(
bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype
)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
# 构造选择的选择的rois
# 获得相对于原始图片大小的rois
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones((sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights
)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(
blob_dict, sampled_boxes, roidb, im_scale, batch_idx
)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(
blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx
)
return blob_dict
def _sample_rois(roidb, im_scale, batch_idx):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM)
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(
fg_inds, size=fg_rois_per_this_image, replace=False
)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where(
(max_overlaps < cfg.TRAIN.BG_THRESH_HI) &
(max_overlaps >= cfg.TRAIN.BG_THRESH_LO)
)[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(
bg_inds, size=bg_rois_per_this_image, replace=False
)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :]
)
bbox_outside_weights = np.array(
bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype
)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones((sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
# Base Fast R-CNN blobs
blob_dict = dict(
labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights
)
# Optionally add Mask R-CNN blobs
if cfg.MODEL.MASK_ON:
mask_rcnn_roi_data.add_mask_rcnn_blobs(
blob_dict, sampled_boxes, roidb, im_scale, batch_idx
)
# Optionally add Keypoint R-CNN blobs
if cfg.MODEL.KEYPOINTS_ON:
keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(
blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx
)
### added by srmani
if cfg.TRAIN.SOFT_SAMPLING_ON:
ss_fg_inds = np.where(max_overlaps[keep_inds] >= 0.5)[0]
ss_bg_inds = np.where(max_overlaps[keep_inds] < 0.5)[0]
label_weights = np.zeros(keep_inds.shape[0], dtype=max_overlaps.dtype)
label_weights[ss_fg_inds] = 1 # high weight for positive instances
sigma1 = 0.25
sigma2 = 50.0
sigma3 = 20.0
label_weights[ss_bg_inds] = sigma1 + (1-sigma1)*np.exp(-sigma2*np.exp(-sigma3 * max_overlaps[keep_inds][ss_bg_inds]))
blob_dict['label_weights'] = label_weights
### end added by srmani
return blob_dict
