def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. """ rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :]) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx) # optionally add Domain Adaptive R-CNN blobs if cfg.TRAIN.DOMAIN_ADAPTATION: if roidb['is_source']: blob_dict['dc_label'] = np.expand_dims(np.ones( blob_dict['labels_int32'].shape, dtype=blob_dict['labels_int32'].dtype), axis=1) blob_dict['label_mask'] = np.full(blob_dict['labels_int32'].shape, True) blob_dict['source_labels_int32'] = blob_dict['labels_int32'] blob_dict['source_bbox_targets'] = blob_dict['bbox_targets'] blob_dict['source_bbox_inside_weights'] = blob_dict[ 'bbox_inside_weights'] blob_dict['source_bbox_outside_weights'] = blob_dict[ 'bbox_outside_weights'] blob_dict['da_label_wide'] = np.ones((1, 1, 200, 400), dtype=np.int32) else: blob_dict['dc_label'] = np.expand_dims(np.zeros( blob_dict['labels_int32'].shape, dtype=blob_dict['labels_int32'].dtype), axis=1) blob_dict['label_mask'] = np.full(blob_dict['labels_int32'].shape, False) blob_dict['da_label_wide'] = np.zeros((1, 1, 200, 400), dtype=np.int32) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. """ rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] gt_inds = np.where(roidb['gt_classes'] > 0)[0] gt_boxes = roidb['boxes'][gt_inds, :] gt_assignments = gt_inds[roidb['box_to_gt_ind_map'][keep_inds]] # [mapped_gt_boxes, max_overlaps] mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5)) mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale mapped_gt_boxes[:, 4] = max_overlaps[keep_inds] mapped_gt_boxes[fg_rois_per_this_image:, :] = 0 bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :]) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights, mapped_gt_boxes=mapped_gt_boxes) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == 1: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == 1: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx, cfg.TRAIN.FG_THRESH) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, label_code): """Generate a random sample of RoIs comprising foreground and background examples. """ rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 action_sampled_labels = roidb['max_classes'][keep_inds] action_sampled_labels[ fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] # code.interact(local=locals()) if 'bbox_targets' not in roidb: gt_inds = np.where(roidb['gt_classes'] > 0)[0] gt_boxes = roidb['boxes'][gt_inds, :] gt_assignments = gt_inds[roidb['box_to_gt_ind_map'][keep_inds]] bbox_targets = _compute_targets(sampled_boxes, gt_boxes[gt_assignments, :], sampled_labels) bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets) # code.interact(local=locals()) else: bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :]) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # new version for hierachy training for idx in range(0, len(sampled_labels)): label_int = sampled_labels[idx] if label_int != 0: sampled_labels[idx] = 1 action_sampled_labels[idx] = 0 multi_label_str = label_code['idx_to_label'][str(label_int)] multi_label = map(int, list(multi_label_str)) for k in range(0, 14): if int(multi_label[k + 1]) == 1: action_sampled_labels[idx] = k + 1 else: sampled_labels[idx] = 0 action_sampled_labels[idx] = 0 # Base Fast R-CNN blobs blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False), multilabels_int32=action_sampled_labels.astype( np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights) # previous version for multilabel training ''' expand_sampled_labels = np.expand_dims(sampled_labels,axis=2) expand_sampled_labels = np.repeat(expand_sampled_labels,81,axis=1) for idx in range(0,len(sampled_labels)): label_int = sampled_labels[idx] if label_int != 0: sampled_labels[idx] = 1 multi_label_str = label_code['idx_to_label'][str(label_int)] multi_label = map(int, list(multi_label_str)) expand_sampled_labels[idx] = multi_label else: sampled_labels[idx] = 0 expand_sampled_labels[idx] = [0]*81 # Base Fast R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), multilabels_int32=expand_sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights ) ''' # Base Fast R-CNN blobs ''' blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights ) ''' # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, stage_num): """Generate a random sample of RoIs comprising foreground and background examples. """ # set improving rcnn iou threshold for cascade rcnn bg_thresh_lo = cfg.TRAIN.BG_THRESH_LO if stage_num == 1: fg_thresh = cfg.TRAIN.FG_THRESH # 0.5 bg_thresh_hi = cfg.TRAIN.BG_THRESH_HI # 0.5 elif stage_num == 2: fg_thresh = 0.6 bg_thresh_hi = 0.6 elif stage_num == 3: fg_thresh = 0.7 bg_thresh_hi = 0.7 rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= fg_thresh)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] sampled_max_overlaps = roidb['max_overlaps'][keep_inds] bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :]) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs if stage_num == 1: blob_dict = dict(labels_int32_1st=sampled_labels.astype(np.int32, copy=False), rois_1st=sampled_rois, bbox_targets_1st=bbox_targets, bbox_inside_weights_1st=bbox_inside_weights, bbox_outside_weights_1st=bbox_outside_weights, max_overlaps_1st=sampled_max_overlaps) if stage_num == 2: blob_dict = dict(labels_int32_2nd=sampled_labels.astype(np.int32, copy=False), rois_2nd=sampled_rois, bbox_targets_2nd=bbox_targets, bbox_inside_weights_2nd=bbox_inside_weights, bbox_outside_weights_2nd=bbox_outside_weights, max_overlaps_2nd=sampled_max_overlaps) if stage_num == 3: blob_dict = dict(labels_int32_3rd=sampled_labels.astype(np.int32, copy=False), rois_3rd=sampled_rois, bbox_targets_3rd=bbox_targets, bbox_inside_weights_3rd=bbox_inside_weights, bbox_outside_weights_3rd=bbox_outside_weights) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs( blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. """ if cfg.REID.PSE_ON: img_labels = np.array([0], dtype=np.float32) attr_img_labels = np.array([0], dtype=np.float32) weight = np.array([0.0], dtype=np.float32) attr_weight = np.array([0.0], dtype=np.float32) gt_inds = np.where(roidb['gt_classes'] > 0)[0] assert len(gt_inds) <= 2, 'Only one ground truth for image is allowed.' gt_classes = roidb['gt_classes'][gt_inds].copy() gt_inds = np.where(roidb['gt_attributions'] > 0)[0] assert len(gt_inds) <= 2, 'Only one ground truth for image is allowed.' gt_attributions = roidb['gt_attributions'][gt_inds].copy() classes_or_attributions = roidb['classes_or_attributions'] for i in range(len(gt_classes)): if classes_or_attributions[i] == 0: img_labels[0] = gt_classes[i] - 1 weight[0] = 1.0 elif classes_or_attributions[i] == 1: attr_img_labels[0] = gt_attributions[i] - 1 attr_weight[0] = cfg.REID.PSE_WEIGHT else: img_labels[0] = gt_classes[i] - 1 weight[0] = 1.0 attr_img_labels[0] = gt_attributions[i] - 1 attr_weight[0] = cfg.REID.PSE_WEIGHT blob_dict = dict( labels_int32=img_labels.astype(np.int32, copy=False), attr_labels_int32=attr_img_labels.astype(np.int32, copy=False), weight=weight.astype(np.float32, copy=False), attr_weight=attr_weight.astype(np.float32, copy=False), ) return blob_dict # Get image label img_labels_oh = np.zeros((1, cfg.MODEL.NUM_CLASSES - 1), dtype=np.float32) img_labels = np.zeros((1), dtype=np.float32) gt_inds = np.where(roidb['gt_classes'] > 0)[0] assert len(gt_inds) == 1, 'Only one ground truth for image is allowed.' gt_classes = roidb['gt_classes'][gt_inds].copy() img_labels_oh[0][gt_classes[0] - 1] = 1 img_labels[0] = gt_classes[0] - 1 blob_dict = dict( labels_int32=img_labels.astype(np.int32, copy=False), labels_oh=img_labels_oh.astype(np.float32, copy=False), ) return blob_dict rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice( fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice( bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :]) bbox_outside_weights = np.array( bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs( blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. """ rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice( fg_inds, size=fg_rois_per_this_image, replace=False ) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where( (max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO) )[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice( bg_inds, size=bg_rois_per_this_image, replace=False ) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :] ) bbox_outside_weights = np.array( bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype ) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones((sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs( blob_dict, sampled_boxes, roidb, im_scale, batch_idx ) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs( blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx ) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, stage): """Generate a random sample of RoIs comprising foreground and background examples. """ fg_thresh = cfg.CASCADE_RCNN.FG_THRESHS[stage - 1] bg_thresh_hi = cfg.CASCADE_RCNN.BG_THRESHS_HI[stage - 1] bg_thresh_lo = cfg.CASCADE_RCNN.BG_THRESHS_LO[stage - 1] max_overlaps = roidb["max_overlaps"] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= fg_thresh)[0] fg_rois_per_this_image = fg_inds.size # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo))[0] # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb["max_classes"][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb["boxes"][keep_inds] gt_inds = np.where(roidb["gt_classes"] > 0)[0] gt_boxes = roidb["boxes"][gt_inds, :] gt_assignments = gt_inds[roidb["box_to_gt_ind_map"][keep_inds]] # [mapped_gt_boxes, max_overlaps] mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5)) mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale mapped_gt_boxes[:, 4] = max_overlaps[keep_inds] mapped_gt_boxes[fg_rois_per_this_image:, :] = 0 if "bbox_targets" not in roidb: bbox_targets = _compute_targets(sampled_boxes, gt_boxes[gt_assignments, :], sampled_labels, stage) else: bbox_targets = roidb['bbox_targets'][keep_inds, :] bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Cascade R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights, mapped_gt_boxes=mapped_gt_boxes, ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == stage: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == stage: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_this_image, fg_inds, im_scale, batch_idx, fg_thresh) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx, stage): # 随机产生包含前景和背景的正负样本,即rois集合 """Generate a random sample of RoIs comprising foreground and background examples. """ fg_thresh = cfg.CASCADE_RCNN.FG_THRESHS[stage - 1] bg_thresh_hi = cfg.CASCADE_RCNN.BG_THRESHS_HI[stage - 1] bg_thresh_lo = cfg.CASCADE_RCNN.BG_THRESHS_LO[stage - 1] #### 这个变量很关键:定义了每一个roi与所有gt-bboxes所对应的最大的iou ??? max_overlaps = roidb["max_overlaps"] # Select foreground RoIs as those with >= FG_THRESH overlap #### 选择最大iou高于fg_thresh的那些rois,作为前景的indexes-->fg_inds fg_inds = np.where(max_overlaps >= fg_thresh)[0] #### 一张img所包含的所有fg_rois数量 fg_rois_per_this_image = fg_inds.size # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) #### 选择那些对应的最大iou值介于(bg_thesh_lo, bg_thresh_hi)的那些rois,采样为背景 bg_inds = np.where((max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo))[0] # The indices that we're selecting (both fg and bg) #### keep_inds为我们将要采样的样本的索引列表(包括正,负样本) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with #### sampled_labels即为roidb中采样的rois所对应的标签值labels sampled_labels = roidb["max_classes"][keep_inds] #### 将采样的负样本所对应的标签值都置为0 sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 #### sampled_boxes即为roidb中采样的所有正负样本,shape为(, 4) sampled_boxes = roidb["boxes"][keep_inds] #### 得到roidb中所有的gt_bbox的index gt_inds = np.where(roidb["gt_classes"] > 0)[0] #### 得到gt_bbox构成的集合,shape为(, 4) gt_boxes = roidb["boxes"][gt_inds, :] #### 这个有点不太理解了,gt_assignments为所有采样的作为正样本的rois所对应的gt_bboxes集合 gt_assignments = gt_inds[roidb["box_to_gt_ind_map"][keep_inds]] # [mapped_gt_boxes, max_overlaps] #### 初始化mapped_gt_boxes变量,保存所有采样的rois所对应的gt_bboxes的坐标( # 疑问:对于负样本也有gt_bbox??? ) mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5)) #### mapped_gt_boxes #### 将原图对应的 mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale mapped_gt_boxes[:, 4] = max_overlaps[keep_inds] ####对于负样本所对应的行(fg_rois_per_this_image之后的)的数据,全部置为0 mapped_gt_boxes[fg_rois_per_this_image:, :] = 0 if "bbox_targets" not in roidb: bbox_targets = _compute_targets(sampled_boxes, gt_boxes[gt_assignments, :], sampled_labels, stage) else: bbox_targets = roidb['bbox_targets'][keep_inds, :] bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Cascade R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights, mapped_gt_boxes=mapped_gt_boxes, ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == stage: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == stage: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_this_image, fg_inds, im_scale, batch_idx, fg_thresh) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. 获得fg和bg roi """ # 512 rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) # 128 fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap # 选择overlaps大于给定值的作为正样本 fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs # fg的数量 fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement # 随机选择fg_rois_per_this_image个正样本 if fg_inds.size > 0: fg_inds = npr.choice( fg_inds, size=fg_rois_per_this_image, replace=False ) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where( (max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO) )[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement # 选择bg目标 if bg_inds.size > 0: bg_inds = npr.choice( bg_inds, size=bg_rois_per_this_image, replace=False ) # The indices that we're selecting (both fg and bg) # 合并fg和bg keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] # bg rois的标签置为0 sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :] ) bbox_outside_weights = np.array( bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype ) # Scale rois and format as (batch_idx, x1, y1, x2, y2) # 构造选择的选择的rois # 获得相对于原始图片大小的rois sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones((sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs( blob_dict, sampled_boxes, roidb, im_scale, batch_idx ) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs( blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx ) return blob_dict
def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. """ rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice( fg_inds, size=fg_rois_per_this_image, replace=False ) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where( (max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO) )[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice( bg_inds, size=bg_rois_per_this_image, replace=False ) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :] ) bbox_outside_weights = np.array( bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype ) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones((sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON: mask_rcnn_roi_data.add_mask_rcnn_blobs( blob_dict, sampled_boxes, roidb, im_scale, batch_idx ) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs( blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx ) ### added by srmani if cfg.TRAIN.SOFT_SAMPLING_ON: ss_fg_inds = np.where(max_overlaps[keep_inds] >= 0.5)[0] ss_bg_inds = np.where(max_overlaps[keep_inds] < 0.5)[0] label_weights = np.zeros(keep_inds.shape[0], dtype=max_overlaps.dtype) label_weights[ss_fg_inds] = 1 # high weight for positive instances sigma1 = 0.25 sigma2 = 50.0 sigma3 = 20.0 label_weights[ss_bg_inds] = sigma1 + (1-sigma1)*np.exp(-sigma2*np.exp(-sigma3 * max_overlaps[keep_inds][ss_bg_inds])) blob_dict['label_weights'] = label_weights ### end added by srmani return blob_dict