def _expand_bbox_targets(bbox_target_data): """Bounding-box regression targets are stored in a compact form in the roidb. This function expands those targets into the 4-of-4*K representation used by the network (i.e. only one class has non-zero targets). The loss weights are similarly expanded. Returns: bbox_target_data (ndarray): N x 4K blob of regression targets bbox_inside_weights (ndarray): N x 4K blob of loss weights """ num_bbox_reg_classes = cfg.MODEL.NUM_CLASSES if cfg.MODEL.CLS_AGNOSTIC_BBOX_REG: num_bbox_reg_classes = 2 # bg and fg clss = bbox_target_data[:, 0] bbox_targets = blob_utils.zeros((clss.size, 4 * num_bbox_reg_classes)) bbox_inside_weights = blob_utils.zeros(bbox_targets.shape) inds = np.where(clss > 0)[0] for ind in inds: cls = int(clss[ind]) start = 4 * cls end = start + 4 bbox_targets[ind, start:end] = bbox_target_data[ind, 1:] bbox_inside_weights[ind, start:end] = (1.0, 1.0, 1.0, 1.0) return bbox_targets, bbox_inside_weights
def _expand_bbox_targets(bbox_target_data): """Bounding-box regression targets are stored in a compact form in the roidb. This function expands those targets into the 4-of-4*K representation used by the network (i.e. only one class has non-zero targets). The loss weights are similarly expanded. Returns: bbox_target_data (ndarray): N x 4K blob of regression targets bbox_inside_weights (ndarray): N x 4K blob of loss weights """ num_bbox_reg_classes = cfg.MODEL.NUM_CLASSES if cfg.MODEL.CLS_AGNOSTIC_BBOX_REG: num_bbox_reg_classes = 2 # bg and fg clss = bbox_target_data[:, 0] bbox_targets = blob_utils.zeros((clss.size, 4 * num_bbox_reg_classes)) bbox_inside_weights = blob_utils.zeros(bbox_targets.shape) inds = np.where(clss > 0)[0] for ind in inds: cls = int(clss[ind]) start = 4 * cls end = start + 4 bbox_targets[ind, start:end] = bbox_target_data[ind, 1:] bbox_inside_weights[ind, start:end] = (1.0, 1.0, 1.0, 1.0) return bbox_targets, bbox_inside_weights
def keypoints_to_heatmap_labels(keypoints, rois): """Encode keypoint location in the target heatmap for use in SoftmaxWithLoss. """ # Maps keypoints from the half-open interval [x1, x2) on continuous image # coordinates to the closed interval [0, HEATMAP_SIZE - 1] on discrete image # coordinates. We use the continuous <-> discrete conversion from Heckbert # 1990 ("What is the coordinate of a pixel?"): d = floor(c) and c = d + 0.5, # where d is a discrete coordinate and c is a continuous coordinate. assert keypoints.shape[2] == cfg.KRCNN.NUM_KEYPOINTS shape = (len(rois), cfg.KRCNN.NUM_KEYPOINTS) heatmaps = blob_utils.zeros(shape) weights = blob_utils.zeros(shape) offset_x = rois[:, 0] offset_y = rois[:, 1] scale_x = cfg.KRCNN.HEATMAP_SIZE / (rois[:, 2] - rois[:, 0]) scale_y = cfg.KRCNN.HEATMAP_SIZE / (rois[:, 3] - rois[:, 1]) for kp in range(keypoints.shape[2]): vis = keypoints[:, 2, kp] > 0 x = keypoints[:, 0, kp].astype(np.float32) y = keypoints[:, 1, kp].astype(np.float32) # Since we use floor below, if a keypoint is exactly on the roi's right # or bottom boundary, we shift it in by eps (conceptually) to keep it in # the ground truth heatmap. x_boundary_inds = np.where(x == rois[:, 2])[0] y_boundary_inds = np.where(y == rois[:, 3])[0] x = (x - offset_x) * scale_x x = np.floor(x) if len(x_boundary_inds) > 0: x[x_boundary_inds] = cfg.KRCNN.HEATMAP_SIZE - 1 y = (y - offset_y) * scale_y y = np.floor(y) if len(y_boundary_inds) > 0: y[y_boundary_inds] = cfg.KRCNN.HEATMAP_SIZE - 1 valid_loc = np.logical_and( np.logical_and(x >= 0, y >= 0), np.logical_and( x < cfg.KRCNN.HEATMAP_SIZE, y < cfg.KRCNN.HEATMAP_SIZE)) valid = np.logical_and(valid_loc, vis) valid = valid.astype(np.int32) lin_ind = y * cfg.KRCNN.HEATMAP_SIZE + x heatmaps[:, kp] = lin_ind * valid weights[:, kp] = valid return heatmaps, weights
def keypoints_to_heatmap_labels(keypoints, rois): """Encode keypoint location in the target heatmap for use in SoftmaxWithLoss. """ # Maps keypoints from the half-open interval [x1, x2) on continuous image # coordinates to the closed interval [0, HEATMAP_SIZE - 1] on discrete image # coordinates. We use the continuous <-> discrete conversion from Heckbert # 1990 ("What is the coordinate of a pixel?"): d = floor(c) and c = d + 0.5, # where d is a discrete coordinate and c is a continuous coordinate. assert keypoints.shape[2] == cfg.KRCNN.NUM_KEYPOINTS shape = (len(rois), cfg.KRCNN.NUM_KEYPOINTS) heatmaps = blob_utils.zeros(shape) weights = blob_utils.zeros(shape) offset_x = rois[:, 0] offset_y = rois[:, 1] scale_x = cfg.KRCNN.HEATMAP_SIZE / (rois[:, 2] - rois[:, 0]) scale_y = cfg.KRCNN.HEATMAP_SIZE / (rois[:, 3] - rois[:, 1]) for kp in range(keypoints.shape[2]): vis = keypoints[:, 2, kp] > 0 x = keypoints[:, 0, kp].astype(np.float32) y = keypoints[:, 1, kp].astype(np.float32) # Since we use floor below, if a keypoint is exactly on the roi's right # or bottom boundary, we shift it in by eps (conceptually) to keep it in # the ground truth heatmap. x_boundary_inds = np.where(x == rois[:, 2])[0] y_boundary_inds = np.where(y == rois[:, 3])[0] x = (x - offset_x) * scale_x x = np.floor(x) if len(x_boundary_inds) > 0: x[x_boundary_inds] = cfg.KRCNN.HEATMAP_SIZE - 1 y = (y - offset_y) * scale_y y = np.floor(y) if len(y_boundary_inds) > 0: y[y_boundary_inds] = cfg.KRCNN.HEATMAP_SIZE - 1 valid_loc = np.logical_and( np.logical_and(x >= 0, y >= 0), np.logical_and( x < cfg.KRCNN.HEATMAP_SIZE, y < cfg.KRCNN.HEATMAP_SIZE)) valid = np.logical_and(valid_loc, vis) valid = valid.astype(np.int32) lin_ind = y * cfg.KRCNN.HEATMAP_SIZE + x heatmaps[:, kp] = lin_ind * valid weights[:, kp] = valid return heatmaps, weights
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): """Add Mask R-CNN specific blobs to the input blob dictionary.""" # Prepare the mask targets by associating one gt mask to each training roi # that has a fg (non-bg) class label. M = cfg.MRCNN.RESOLUTION input_w = roidb['input_width'] input_h = roidb['input_height'] polys_gt_inds = np.where((roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0))[0] polys_gt = [roidb['segms'][i] for i in polys_gt_inds] boxes_from_polys = segm_utils.polys_to_boxes(polys_gt) fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = blobs['labels_int32'].copy() roi_has_mask[roi_has_mask > 0] = 1 mask_fg_rois_per_this_image = cfg.MRCNN.MAX_ROIS_PER_IM if fg_inds.shape[0] > 0: if fg_inds.size > mask_fg_rois_per_this_image: fg_inds = np.random.choice(fg_inds, size=mask_fg_rois_per_this_image, replace=False) # Class labels for the foreground rois mask_class_labels = blobs['labels_int32'][fg_inds] masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) all_person_masks = np.zeros( (int(input_h / im_scale), int(input_w / im_scale)), dtype=np.float32) # Find overlap between all foreground rois and the bounding boxes # enclosing each segmentation rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) # Map from each fg rois to the index of the mask with highest overlap # (measured by bbox overlap) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) # add fg targets for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] poly_gt = polys_gt[fg_polys_ind] roi_fg = rois_fg[i] # Rasterize the portion of the polygon mask within the given fg roi # to an M x M binary image mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M) mask = np.array(mask > 0, dtype=np.int32) # Ensure it's binary masks[i, :] = np.reshape(mask, M**2) # to an box_h x box_w binary image mask_wrt_bbox = segm_utils.convert_polys_to_mask_wrt_box( poly_gt, roi_fg) start_y, start_x = int(roi_fg[1]), int(roi_fg[0]) end_y, end_x = start_y + mask_wrt_bbox.shape[ 0], start_x + mask_wrt_bbox.shape[1] all_person_masks[start_y:end_y, start_x:end_x] = mask_wrt_bbox proposal_all_mask = blob_utils.zeros((fg_inds.shape[0], M, M), int32=True) for i in range(rois_fg.shape[0]): roi_fg = rois_fg[i] w = roi_fg[2] - roi_fg[0] h = roi_fg[3] - roi_fg[1] w = int(np.maximum(w, 1)) h = int(np.maximum(h, 1)) proposal_mask = all_person_masks[int(roi_fg[1]):int(roi_fg[1]) + h, int(roi_fg[0]):int(roi_fg[0]) + w] # proposal_mask = proposal_mask.astype(np.float32) proposal_mask = cv2.resize(proposal_mask, (M, M)) proposal_mask = (proposal_mask > 0.5).astype(np.int32) proposal_all_mask[i] = proposal_mask else: # If there are no fg masks (it does happen) # The network cannot handle empty blobs, so we must provide a mask # We simply take the first bg roi, given it an all -1's mask (ignore # label), and label it with class zero (bg). bg_inds = np.where(blobs['labels_int32'] == 0)[0] # rois_fg is actually one background roi, but that's ok because ... rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) # We give it an -1's blob (ignore label) masks = -blob_utils.ones((1, M**2), int32=True) # We label it with class = 0 (background) mask_class_labels = blob_utils.zeros((1, )) # Mark that the first roi has a mask roi_has_mask[0] = 1 proposal_all_mask = -blob_utils.ones((1, M, M), int32=True) if cfg.MRCNN.CLS_SPECIFIC_MASK: masks = _expand_to_class_specific_mask_targets(masks, mask_class_labels) # Scale rois_fg and format as (batch_idx, x1, y1, x2, y2) rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # Update blobs dict with Mask R-CNN blobs blobs['mask_rois'] = rois_fg blobs['roi_has_mask_int32'] = roi_has_mask blobs['masks_int32'] = masks # blobs['mask_labels'] = np.argmax(masks.reshape((-1,cfg.MODEL.NUM_CLASSES,M,M)),axis=1).reshape((-1,M,M)).astype(np.int32) # blobs['mask_weights'] = np.ones(blobs['mask_labels'].shape, dtype=np.float32) # add by wxh if cfg.MRCNN.USE_CLS_EMBS: fg_embs, bg_embs, fg_weights, bg_weights = masks_to_embs( masks.reshape((-1, cfg.MODEL.NUM_CLASSES, M, M))) # print('fg',fg_embs.max(), fg_embs.min()) # print('bg',bg_embs.max(), bg_embs.min()) fg_norms = np.sum(fg_embs, axis=(1, 2)) fg_norms[fg_norms != 0] = 28. * 28. / (fg_norms[fg_norms != 0] + 1e-6) bg_norms = np.sum(bg_embs, axis=(1, 2)) bg_norms[bg_norms != 0] = 28. * 28. / (bg_norms[bg_norms != 0] + 1e-6) blobs['fg_mask'] = np.repeat(np.reshape(fg_embs, (-1, 1, M, M)), 2, axis=1) blobs['bg_mask'] = np.repeat(np.reshape(bg_embs, (-1, 1, M, M)), 2, axis=1) blobs['fg_norm'] = np.repeat(np.reshape(fg_norms, (-1, 1)), 2, axis=1) blobs['bg_norm'] = np.repeat(np.reshape(bg_norms, (-1, 1)), 2, axis=1) blobs['mask_emb_fg_labels'] = np.ones((fg_embs.shape[0], 1), dtype=np.int32) blobs['mask_emb_bg_labels'] = np.zeros((bg_embs.shape[0], 1), dtype=np.int32) # blobs['mask_emb_weights'] = np.vstack([fg_weights, bg_weights]).reshape((-1,1)).astype(np.float32) if cfg.MRCNN.BBOX_CASCADE_MASK_ON: blobs['inter_masks_int32'] = proposal_all_mask
def keypoints_to_global_heatmap_labels(keypoints, rois, input_h, input_w): assert keypoints.shape[2] == cfg.KRCNN.NUM_KEYPOINTS M = cfg.KRCNN.HEATMAP_SIZE shape = (len(rois), cfg.KRCNN.NUM_KEYPOINTS) heatmaps = blob_utils.zeros(shape) weights = blob_utils.zeros(shape) global_heatmaps = np.zeros((input_h, input_w, cfg.KRCNN.NUM_KEYPOINTS), dtype=np.float32) offset_x = rois[:, 0] offset_y = rois[:, 1] scale_x = cfg.KRCNN.HEATMAP_SIZE / (rois[:, 2] - rois[:, 0]) scale_y = cfg.KRCNN.HEATMAP_SIZE / (rois[:, 3] - rois[:, 1]) for kp in range(keypoints.shape[2]): vis = keypoints[:, 2, kp] > 0 x = keypoints[:, 0, kp].astype(np.float32) y = keypoints[:, 1, kp].astype(np.float32) # valid_loc = np.logical_and(x<input_w,y<input_h) # vis = np.logical_and(vis, valid_loc) # valid_x = x[vis].astype(np.int32) # valid_y = y[vis].astype(np.int32) # for i_loc in range(valid_x.shape[0]): # mu_x = valid_x[i_loc] # mu_y = valid_y[i_loc] # range_x = np.arange(mu_x - 1, mu_x + 2) # range_y = np.arange(mu_y - 1, mu_y + 2) # range_x[range_x<0] = 0 # range_x[range_x>=input_w]=input_w-1 # range_y[range_y<1] = 0 # range_y[range_y>=input_h]=input_h-1 # global_heatmaps[range_y, range_x, kp] = 1 # Since we use floor below, if a keypoint is exactly on the roi's right # or bottom boundary, we shift it in by eps (conceptually) to keep it in # the ground truth heatmap. x_boundary_inds = np.where(x == rois[:, 2])[0] y_boundary_inds = np.where(y == rois[:, 3])[0] x = (x - offset_x) * scale_x x = np.floor(x) if len(x_boundary_inds) > 0: x[x_boundary_inds] = cfg.KRCNN.HEATMAP_SIZE - 1 y = (y - offset_y) * scale_y y = np.floor(y) if len(y_boundary_inds) > 0: y[y_boundary_inds] = cfg.KRCNN.HEATMAP_SIZE - 1 valid_loc = np.logical_and( np.logical_and(x >= 0, y >= 0), np.logical_and(x < cfg.KRCNN.HEATMAP_SIZE, y < cfg.KRCNN.HEATMAP_SIZE)) valid = np.logical_and(valid_loc, vis) valid_x = x[valid] / scale_x[valid] + offset_x[valid] valid_x = valid_x.astype(np.int32) valid_y = y[valid] / scale_y[valid] + offset_y[valid] valid_y = valid_y.astype(np.int32) global_heatmaps[valid_y, valid_x, kp] = 1 valid = valid.astype(np.int32) lin_ind = y * cfg.KRCNN.HEATMAP_SIZE + x heatmaps[:, kp] = lin_ind * valid weights[:, kp] = valid proposal_all_heatmaps = blob_utils.zeros( (rois.shape[0], cfg.KRCNN.NUM_KEYPOINTS, M, M), int32=True) for i in range(rois.shape[0]): roi_fg = rois[i] w = roi_fg[2] - roi_fg[0] h = roi_fg[3] - roi_fg[1] w = int(np.maximum(w, 1)) h = int(np.maximum(h, 1)) proposal_hm = global_heatmaps[int(roi_fg[1]):int(roi_fg[1]) + h, int(roi_fg[0]):int(roi_fg[0]) + w, :] proposal_hm = cv2.resize(proposal_hm, (M, M)) proposal_hm = (proposal_hm > 0.5).astype(np.int32) proposal_all_heatmaps[i] = proposal_hm.transpose((2, 0, 1)) return heatmaps, weights, proposal_all_heatmaps
def _sample_rois(roidb, im_scale, batch_idx, stage): """Generate a random sample of RoIs comprising foreground and background examples. """ fg_thresh = cfg.CASCADE_RCNN.FG_THRESHS[stage - 1] bg_thresh_hi = cfg.CASCADE_RCNN.BG_THRESHS_HI[stage - 1] bg_thresh_lo = cfg.CASCADE_RCNN.BG_THRESHS_LO[stage - 1] max_overlaps = roidb["max_overlaps"] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= fg_thresh)[0] fg_rois_per_this_image = fg_inds.size # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo))[0] # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb["max_classes"][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb["boxes"][keep_inds] gt_inds = np.where(roidb["gt_classes"] > 0)[0] gt_boxes = roidb["boxes"][gt_inds, :] gt_assignments = gt_inds[roidb["box_to_gt_ind_map"][keep_inds]] # [mapped_gt_boxes, max_overlaps] mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5)) mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale mapped_gt_boxes[:, 4] = max_overlaps[keep_inds] mapped_gt_boxes[fg_rois_per_this_image:, :] = 0 if "bbox_targets" not in roidb: bbox_targets = _compute_targets(sampled_boxes, gt_boxes[gt_assignments, :], sampled_labels, stage) else: bbox_targets = roidb['bbox_targets'][keep_inds, :] bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Cascade R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights, mapped_gt_boxes=mapped_gt_boxes, ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == stage: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == stage: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_this_image, fg_inds, im_scale, batch_idx, fg_thresh) return blob_dict
def add_rpn_blobs(blobs, im_scales, roidb): """Add blobs needed training RPN-only and end-to-end Faster R-CNN models.""" if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper k_max = cfg.FPN.RPN_MAX_LEVEL k_min = cfg.FPN.RPN_MIN_LEVEL foas = [] # fetch the spatial scales for every FPN level except fpn6 #fpn_spatial_scales = globals().get('fpn_level_info_' + cfg.MODEL.BACKBONE_NAME + '_conv5')().spatial_scales fpn_spatial_scales = getattr(FPN, 'fpn_level_info_' + cfg.MODEL.BACKBONE_NAME + '_conv5')().spatial_scales for lvl in range(k_min, k_max): field_stride = 1. / fpn_spatial_scales[k_max-lvl-1] #field_stride = 2.**(lvl - int(math.log(cfg.FPN.FINEST_LEVEL_SCALE,2))-2) anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ) anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS foa = data_utils.get_field_of_anchors( field_stride, anchor_sizes, anchor_aspect_ratios ) foas.append(foa) # for p6, the scale should be the corest level divided by 2 if k_max == 6: field_stride = 2 * (1. / fpn_spatial_scales[0]) anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(k_max - k_min), ) anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS foa = data_utils.get_field_of_anchors( field_stride, anchor_sizes, anchor_aspect_ratios ) foas.append(foa) all_anchors = np.concatenate([f.field_of_anchors for f in foas]) else: foa = data_utils.get_field_of_anchors( cfg.RPN.STRIDE, cfg.RPN.SIZES, cfg.RPN.ASPECT_RATIOS ) all_anchors = foa.field_of_anchors for im_i, entry in enumerate(roidb): scale = im_scales[im_i] im_height = np.round(entry['height'] * scale) im_width = np.round(entry['width'] * scale) gt_inds = np.where( (entry['gt_classes'] > 0) & (entry['is_crowd'] == 0) )[0] gt_rois = entry['boxes'][gt_inds, :] * scale # TODO(rbg): gt_boxes is poorly named; # should be something like 'gt_rois_info' gt_boxes = blob_utils.zeros((len(gt_inds), 6)) gt_boxes[:, 0] = im_i # batch inds gt_boxes[:, 1:5] = gt_rois gt_boxes[:, 5] = entry['gt_classes'][gt_inds] im_info = np.array([[im_height, im_width, scale]], dtype=np.float32) blobs['im_info'].append(im_info) # Add RPN targets if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper rpn_blobs = _get_rpn_blobs( im_height, im_width, foas, all_anchors, gt_rois ) for i, lvl in enumerate(range(k_min, k_max + 1)): for k, v in rpn_blobs[i].items(): blobs[k + '_fpn' + str(lvl)].append(v) else: # Classical RPN, applied to a single feature level rpn_blobs = _get_rpn_blobs( im_height, im_width, [foa], all_anchors, gt_rois ) for k, v in rpn_blobs.items(): blobs[k].append(v) for k, v in blobs.items(): if isinstance(v, list) and len(v) > 0: blobs[k] = np.concatenate(v) valid_keys = [ 'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes', 'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints' ] minimal_roidb = [{} for _ in range(len(roidb))] for i, e in enumerate(roidb): for k in valid_keys: if k in e: minimal_roidb[i][k] = e[k] blobs['roidb'] = blob_utils.serialize(minimal_roidb) # Always return valid=True, since RPN minibatches are valid by design return True
def _sample_rois(roidb, im_scale, batch_idx): """Generate a random sample of RoIs comprising foreground and background examples. """ rois_per_image = int(cfg.TRAIN.BATCH_SIZE_PER_IM) fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)) max_overlaps = roidb['max_overlaps'] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size) # Sample foreground regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with sampled_labels = roidb['max_classes'][keep_inds] sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 sampled_boxes = roidb['boxes'][keep_inds] gt_inds = np.where(roidb['gt_classes'] > 0)[0] gt_boxes = roidb['boxes'][gt_inds, :] gt_assignments = gt_inds[roidb['box_to_gt_ind_map'][keep_inds]] # [mapped_gt_boxes, max_overlaps] mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5)) mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale mapped_gt_boxes[:, 4] = max_overlaps[keep_inds] mapped_gt_boxes[fg_rois_per_this_image:, :] = 0 bbox_targets, bbox_inside_weights = _expand_bbox_targets( roidb['bbox_targets'][keep_inds, :]) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Fast R-CNN blobs blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights, mapped_gt_boxes=mapped_gt_boxes) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == 1: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == 1: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_image, fg_inds, im_scale, batch_idx, cfg.TRAIN.FG_THRESH) return blob_dict
def add_rpn_blobs(blobs, im_scales, roidb): """Add blobs needed training RPN-only and end-to-end Faster R-CNN models.""" if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper k_max = cfg.FPN.RPN_MAX_LEVEL k_min = cfg.FPN.RPN_MIN_LEVEL foas = [] for lvl in range(k_min, k_max + 1): field_stride = 2.**lvl anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ) anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS foa = data_utils.get_field_of_anchors(field_stride, anchor_sizes, anchor_aspect_ratios) foas.append(foa) all_anchors = np.concatenate([f.field_of_anchors for f in foas]) else: foa = data_utils.get_field_of_anchors(cfg.RPN.STRIDE, cfg.RPN.SIZES, cfg.RPN.ASPECT_RATIOS) all_anchors = foa.field_of_anchors for im_i, entry in enumerate(roidb): scale = im_scales[im_i] input_data = blobs['data'][im_i] # print(input_data.shape) im_height = np.round(entry['height'] * scale) im_width = np.round(entry['width'] * scale) entry['input_width'] = input_data.shape[2] entry['input_height'] = input_data.shape[1] gt_inds = np.where((entry['gt_classes'] > 0) & (entry['is_crowd'] == 0))[0] gt_rois = entry['boxes'][gt_inds, :] * scale # TODO(rbg): gt_boxes is poorly named; # should be something like 'gt_rois_info' gt_boxes = blob_utils.zeros((len(gt_inds), 6)) gt_boxes[:, 0] = im_i # batch inds gt_boxes[:, 1:5] = gt_rois gt_boxes[:, 5] = entry['gt_classes'][gt_inds] im_info = np.array([[im_height, im_width, scale]], dtype=np.float32) blobs['im_info'].append(im_info) # Add RPN targets if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper rpn_blobs = _get_rpn_blobs(im_height, im_width, foas, all_anchors, gt_rois) for i, lvl in enumerate(range(k_min, k_max + 1)): for k, v in rpn_blobs[i].items(): blobs[k + '_fpn' + str(lvl)].append(v) else: # Classical RPN, applied to a single feature level rpn_blobs = _get_rpn_blobs(im_height, im_width, [foa], all_anchors, gt_rois) for k, v in rpn_blobs.items(): blobs[k].append(v) for k, v in blobs.items(): if isinstance(v, list) and len(v) > 0: blobs[k] = np.concatenate(v) valid_keys = [ 'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes', 'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints', 'flipped', 'ignore_UV_body', 'dp_x', 'dp_y', 'dp_I', 'dp_U', 'dp_V', 'dp_masks', 'input_width', 'input_height' ] minimal_roidb = [{} for _ in range(len(roidb))] for i, e in enumerate(roidb): for k in valid_keys: if k in e: minimal_roidb[i][k] = e[k] blobs['roidb'] = blob_utils.serialize(minimal_roidb) # Always return valid=True, since RPN minibatches are valid by design return True
def _sample_rois(roidb, im_scale, batch_idx, stage): # 随机产生包含前景和背景的正负样本,即rois集合 """Generate a random sample of RoIs comprising foreground and background examples. """ fg_thresh = cfg.CASCADE_RCNN.FG_THRESHS[stage - 1] bg_thresh_hi = cfg.CASCADE_RCNN.BG_THRESHS_HI[stage - 1] bg_thresh_lo = cfg.CASCADE_RCNN.BG_THRESHS_LO[stage - 1] #### 这个变量很关键:定义了每一个roi与所有gt-bboxes所对应的最大的iou ??? max_overlaps = roidb["max_overlaps"] # Select foreground RoIs as those with >= FG_THRESH overlap #### 选择最大iou高于fg_thresh的那些rois,作为前景的indexes-->fg_inds fg_inds = np.where(max_overlaps >= fg_thresh)[0] #### 一张img所包含的所有fg_rois数量 fg_rois_per_this_image = fg_inds.size # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) #### 选择那些对应的最大iou值介于(bg_thesh_lo, bg_thresh_hi)的那些rois,采样为背景 bg_inds = np.where((max_overlaps < bg_thresh_hi) & (max_overlaps >= bg_thresh_lo))[0] # The indices that we're selecting (both fg and bg) #### keep_inds为我们将要采样的样本的索引列表(包括正,负样本) keep_inds = np.append(fg_inds, bg_inds) # Label is the class each RoI has max overlap with #### sampled_labels即为roidb中采样的rois所对应的标签值labels sampled_labels = roidb["max_classes"][keep_inds] #### 将采样的负样本所对应的标签值都置为0 sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0 #### sampled_boxes即为roidb中采样的所有正负样本,shape为(, 4) sampled_boxes = roidb["boxes"][keep_inds] #### 得到roidb中所有的gt_bbox的index gt_inds = np.where(roidb["gt_classes"] > 0)[0] #### 得到gt_bbox构成的集合,shape为(, 4) gt_boxes = roidb["boxes"][gt_inds, :] #### 这个有点不太理解了,gt_assignments为所有采样的作为正样本的rois所对应的gt_bboxes集合 gt_assignments = gt_inds[roidb["box_to_gt_ind_map"][keep_inds]] # [mapped_gt_boxes, max_overlaps] #### 初始化mapped_gt_boxes变量,保存所有采样的rois所对应的gt_bboxes的坐标( # 疑问:对于负样本也有gt_bbox??? ) mapped_gt_boxes = blob_utils.zeros((keep_inds.size, 5)) #### mapped_gt_boxes #### 将原图对应的 mapped_gt_boxes[:, :4] = gt_boxes[gt_assignments, :] * im_scale mapped_gt_boxes[:, 4] = max_overlaps[keep_inds] ####对于负样本所对应的行(fg_rois_per_this_image之后的)的数据,全部置为0 mapped_gt_boxes[fg_rois_per_this_image:, :] = 0 if "bbox_targets" not in roidb: bbox_targets = _compute_targets(sampled_boxes, gt_boxes[gt_assignments, :], sampled_labels, stage) else: bbox_targets = roidb['bbox_targets'][keep_inds, :] bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets) bbox_outside_weights = np.array(bbox_inside_weights > 0, dtype=bbox_inside_weights.dtype) # Scale rois and format as (batch_idx, x1, y1, x2, y2) sampled_rois = sampled_boxes * im_scale repeated_batch_idx = batch_idx * blob_utils.ones( (sampled_rois.shape[0], 1)) sampled_rois = np.hstack((repeated_batch_idx, sampled_rois)) # Base Cascade R-CNN blobs blob_dict = dict( labels_int32=sampled_labels.astype(np.int32, copy=False), rois=sampled_rois, bbox_targets=bbox_targets, bbox_inside_weights=bbox_inside_weights, bbox_outside_weights=bbox_outside_weights, mapped_gt_boxes=mapped_gt_boxes, ) # Optionally add Mask R-CNN blobs if cfg.MODEL.MASK_ON and cfg.MRCNN.AT_STAGE == stage: mask_rcnn_roi_data.add_mask_rcnn_blobs(blob_dict, sampled_boxes, roidb, im_scale, batch_idx) # Optionally add Keypoint R-CNN blobs if cfg.MODEL.KEYPOINTS_ON and cfg.KRCNN.AT_STAGE == stage: keypoint_rcnn_roi_data.add_keypoint_rcnn_blobs(blob_dict, roidb, fg_rois_per_this_image, fg_inds, im_scale, batch_idx, fg_thresh) return blob_dict
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): """Add Mask R-CNN specific blobs to the input blob dictionary.""" # Prepare the mask targets by associating one gt mask to each training roi # that has a fg (non-bg) class label. M = cfg.MRCNN.RESOLUTION polys_gt_inds = np.where( (roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0) )[0] polys_gt = [roidb['segms'][i] for i in polys_gt_inds] boxes_from_polys = segm_utils.polys_to_boxes(polys_gt) fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = blobs['labels_int32'].copy() roi_has_mask[roi_has_mask > 0] = 1 if fg_inds.shape[0] > 0: # Class labels for the foreground rois mask_class_labels = blobs['labels_int32'][fg_inds] masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) # Find overlap between all foreground rois and the bounding boxes # enclosing each segmentation rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False) ) # Map from each fg rois to the index of the mask with highest overlap # (measured by bbox overlap) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) # add fg targets for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] poly_gt = polys_gt[fg_polys_ind] roi_fg = rois_fg[i] # Rasterize the portion of the polygon mask within the given fg roi # to an M x M binary image mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M) mask = np.array(mask > 0, dtype=np.int32) # Ensure it's binary masks[i, :] = np.reshape(mask, M**2) else: # If there are no fg masks (it does happen) # The network cannot handle empty blobs, so we must provide a mask # We simply take the first bg roi, given it an all -1's mask (ignore # label), and label it with class zero (bg). bg_inds = np.where(blobs['labels_int32'] == 0)[0] # rois_fg is actually one background roi, but that's ok because ... rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) # We give it an -1's blob (ignore label) masks = -blob_utils.ones((1, M**2), int32=True) # We label it with class = 0 (background) mask_class_labels = blob_utils.zeros((1, )) # Mark that the first roi has a mask roi_has_mask[0] = 1 if cfg.MRCNN.CLS_SPECIFIC_MASK: masks = _expand_to_class_specific_mask_targets(masks, mask_class_labels) # Scale rois_fg and format as (batch_idx, x1, y1, x2, y2) rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # Update blobs dict with Mask R-CNN blobs blobs['mask_rois'] = rois_fg blobs['roi_has_mask_int32'] = roi_has_mask blobs['masks_int32'] = masks
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): """Add Mask R-CNN specific blobs to the input blob dictionary.""" # Prepare the mask targets by associating one gt mask to each training roi # that has a fg (non-bg) class label. M = cfg.MRCNN.RESOLUTION polys_gt_inds = np.where( (roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0) )[0] polys_gt = [roidb['segms'][i] for i in polys_gt_inds] boxes_from_polys = segm_utils.polys_to_boxes(polys_gt) fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = blobs['labels_int32'].copy() roi_has_mask[roi_has_mask > 0] = 1 if fg_inds.shape[0] > 0: # Class labels for the foreground rois mask_class_labels = blobs['labels_int32'][fg_inds] masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) # Find overlap between all foreground rois and the bounding boxes # enclosing each segmentation rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False) ) # Map from each fg rois to the index of the mask with highest overlap # (measured by bbox overlap) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) # add fg targets for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] poly_gt = polys_gt[fg_polys_ind] roi_fg = rois_fg[i] # Rasterize the portion of the polygon mask within the given fg roi # to an M x M binary image mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M) mask = np.array(mask > 0, dtype=np.int32) # Ensure it's binary masks[i, :] = np.reshape(mask, M**2) else: # If there are no fg masks (it does happen) # The network cannot handle empty blobs, so we must provide a mask # We simply take the first bg roi, given it an all -1's mask (ignore # label), and label it with class zero (bg). bg_inds = np.where(blobs['labels_int32'] == 0)[0] # rois_fg is actually one background roi, but that's ok because ... rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) # We give it an -1's blob (ignore label) masks = -blob_utils.ones((1, M**2), int32=True) # We label it with class = 0 (background) mask_class_labels = blob_utils.zeros((1, )) # Mark that the first roi has a mask roi_has_mask[0] = 1 if cfg.MRCNN.CLS_SPECIFIC_MASK: masks = _expand_to_class_specific_mask_targets(masks, mask_class_labels) # Scale rois_fg and format as (batch_idx, x1, y1, x2, y2) rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # Update blobs dict with Mask R-CNN blobs blobs['mask_rois'] = rois_fg blobs['roi_has_mask_int32'] = roi_has_mask blobs['masks_int32'] = masks
def add_rpn_blobs(blobs, im_scales, roidb): """Add blobs needed training RPN-only and end-to-end Faster R-CNN models.""" """ 添加训练faster rcnn需要的blobs """ if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper k_max = cfg.FPN.RPN_MAX_LEVEL k_min = cfg.FPN.RPN_MIN_LEVEL foas = [] for lvl in range(k_min, k_max + 1): # lvl = 2 => 4.0 field_stride = 2.**lvl # 32.0 anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ) # [0.5, 1.0, 2.0] anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS # 对于一个特征图,获得特征图上每一个cell所对应的anchor, # 该anchor对应于网络输入的大小 foa = data_utils.get_field_of_anchors(field_stride, anchor_sizes, anchor_aspect_ratios) foas.append(foa) all_anchors = np.concatenate([f.field_of_anchors for f in foas]) else: foa = data_utils.get_field_of_anchors(cfg.RPN.STRIDE, cfg.RPN.SIZES, cfg.RPN.ASPECT_RATIOS) all_anchors = foa.field_of_anchors for im_i, entry in enumerate(roidb): scale = im_scales[im_i] # * scale获得相对于网络输入的信息 im_height = np.round(entry['height'] * scale) im_width = np.round(entry['width'] * scale) gt_inds = np.where((entry['gt_classes'] > 0) & (entry['is_crowd'] == 0))[0] # gt box gt_rois = entry['boxes'][gt_inds, :] * scale # TODO(rbg): gt_boxes is poorly named; # should be something like 'gt_rois_info' gt_boxes = blob_utils.zeros((len(gt_inds), 6)) # 属于哪个图片 gt_boxes[:, 0] = im_i # batch inds # box gt_boxes[:, 1:5] = gt_rois # 类别信息 gt_boxes[:, 5] = entry['gt_classes'][gt_inds] # 写入blob im_info = np.array([[im_height, im_width, scale]], dtype=np.float32) blobs['im_info'].append(im_info) # Add RPN targets # 添加RPN的目标值 if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper rpn_blobs = _get_rpn_blobs(im_height, im_width, foas, all_anchors, gt_rois) for i, lvl in enumerate(range(k_min, k_max + 1)): for k, v in rpn_blobs[i].items(): blobs[k + '_fpn' + str(lvl)].append(v) else: # Classical RPN, applied to a single feature level rpn_blobs = _get_rpn_blobs(im_height, im_width, [foa], all_anchors, gt_rois) for k, v in rpn_blobs.items(): blobs[k].append(v) for k, v in blobs.items(): if isinstance(v, list) and len(v) > 0: blobs[k] = np.concatenate(v) valid_keys = [ 'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes', 'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints' ] minimal_roidb = [{} for _ in range(len(roidb))] for i, e in enumerate(roidb): for k in valid_keys: if k in e: minimal_roidb[i][k] = e[k] blobs['roidb'] = blob_utils.serialize(minimal_roidb) # Always return valid=True, since RPN minibatches are valid by design return True
def add_rpn_blobs(blobs, im_scales, roidb): """Add blobs needed training RPN-only and end-to-end Faster R-CNN models.""" if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper k_max = cfg.FPN.RPN_MAX_LEVEL k_min = cfg.FPN.RPN_MIN_LEVEL foas = [] for lvl in range(k_min, k_max + 1): field_stride = 2.**lvl anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ) anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS foa = data_utils.get_field_of_anchors( field_stride, anchor_sizes, anchor_aspect_ratios ) foas.append(foa) all_anchors = np.concatenate([f.field_of_anchors for f in foas]) else: foa = data_utils.get_field_of_anchors( cfg.RPN.STRIDE, cfg.RPN.SIZES, cfg.RPN.ASPECT_RATIOS ) all_anchors = foa.field_of_anchors for im_i, entry in enumerate(roidb): scale = im_scales[im_i] im_height = np.round(entry['height'] * scale) im_width = np.round(entry['width'] * scale) gt_inds = np.where( (entry['gt_classes'] > 0) & (entry['is_crowd'] == 0) )[0] gt_rois = entry['boxes'][gt_inds, :] * scale # TODO(rbg): gt_boxes is poorly named; # should be something like 'gt_rois_info' gt_boxes = blob_utils.zeros((len(gt_inds), 6)) gt_boxes[:, 0] = im_i # batch inds gt_boxes[:, 1:5] = gt_rois gt_boxes[:, 5] = entry['gt_classes'][gt_inds] im_info = np.array([[im_height, im_width, scale]], dtype=np.float32) blobs['im_info'].append(im_info) # Add RPN targets if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN: # RPN applied to many feature levels, as in the FPN paper rpn_blobs = _get_rpn_blobs( im_height, im_width, foas, all_anchors, gt_rois ) for i, lvl in enumerate(range(k_min, k_max + 1)): for k, v in rpn_blobs[i].items(): blobs[k + '_fpn' + str(lvl)].append(v) else: # Classical RPN, applied to a single feature level rpn_blobs = _get_rpn_blobs( im_height, im_width, [foa], all_anchors, gt_rois ) for k, v in rpn_blobs.items(): blobs[k].append(v) for k, v in blobs.items(): if isinstance(v, list) and len(v) > 0: blobs[k] = np.concatenate(v) valid_keys = [ 'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes', 'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints','flipped', 'ignore_UV_body','dp_x','dp_y','dp_I','dp_U','dp_V','dp_masks' ] minimal_roidb = [{} for _ in range(len(roidb))] for i, e in enumerate(roidb): for k in valid_keys: if k in e: minimal_roidb[i][k] = e[k] blobs['roidb'] = blob_utils.serialize(minimal_roidb) # Always return valid=True, since RPN minibatches are valid by design return True
def add_mask_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): """Add Mask R-CNN specific blobs to the input blob dictionary.""" # Prepare the mask targets by associating one gt mask to each training roi # that has a fg (non-bg) class label. M = cfg.MRCNN.RESOLUTION # gao 6,29 gt_inds = np.where((roidb['gt_classes'] > 0) & (roidb['is_crowd'] == 0))[0] boxes_from_polys = roidb['boxes'][gt_inds, :] gt_classes = roidb['gt_classes'][gt_inds] im_label = cv2.imread(roidb['ins_seg'], 0) if roidb['flipped'] == 1: # convert flipped label to original im_label = im_label[:, ::-1] dataset_name = cfg.TRAIN.DATASETS[0] if 'LIP' in dataset_name: flipped_2_orig_class = { 14: 15, 15: 14, 16: 17, 17: 16, 18: 19, 19: 18 } if 'ATR' in dataset_name: flipped_2_orig_class = { 9: 10, 10: 9, 12: 13, 13: 12, 14: 15, 15: 14 } gt_classes_ = copy.deepcopy(gt_classes) for i in flipped_2_orig_class.keys(): index_i = np.where(gt_classes_ == i)[0] if len(index_i) == 0: continue gt_classes[index_i] = flipped_2_orig_class[i] # gt_inds_flip = np.where(gt_classes>13)[0] # for i in gt_inds_flip: # gt_classes[i] = flipped_2_orig_class[gt_classes[i]] fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = blobs['labels_int32'].copy() roi_has_mask[roi_has_mask > 0] = 1 if fg_inds.shape[0] > 0: # Class labels for the foreground rois mask_class_labels = blobs['labels_int32'][fg_inds] masks = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) # Find overlap between all foreground rois and the bounding boxes # enclosing each segmentation rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) # Map from each fg rois to the index of the mask with highest overlap # (measured by bbox overlap) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) # add fg targets for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] # poly_gt = polys_gt[fg_polys_ind] roi_fg = rois_fg[i] # Rasterize the portion of the polygon mask within the given fg roi # to an M x M binary image #logger.info('roi_fg, label shape: {},{}'.format(roi_fg,im_label.shape)) x0, y0, x1, y1 = roi_fg x0 = min(int(x0), im_label.shape[1]) x1 = min(int(x1 + 1), im_label.shape[1]) y0 = min(int(y0), im_label.shape[0]) y1 = min(int(y1 + 1), im_label.shape[0]) #logger.info('x0,y0,x1,y1: {}'.format(x0, y0, x1, y1)) mask_ = im_label[y0:y1, x0:x1] #logger.info('mask_ shape: {}, gt_classes[fg_polys_ind]:{}'.format(mask_.shape, boxes_from_polys[fg_polys_ind])) # mask = segm_utils.polys_to_mask_wrt_box(poly_gt, roi_fg, M) mask = np.array(mask_ == gt_classes[fg_polys_ind], dtype=np.int32) # Ensure it's binary mask = cv2.resize(mask, (M, M), interpolation=cv2.INTER_NEAREST) masks[i, :] = np.reshape(mask, M**2) im_label = None else: # If there are no fg masks (it does happen) # The network cannot handle empty blobs, so we must provide a mask # We simply take the first bg roi, given it an all -1's mask (ignore # label), and label it with class zero (bg). bg_inds = np.where(blobs['labels_int32'] == 0)[0] # rois_fg is actually one background roi, but that's ok because ... rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) # We give it an -1's blob (ignore label) masks = -blob_utils.ones((1, M**2), int32=True) # We label it with class = 0 (background) mask_class_labels = blob_utils.zeros((1, )) # Mark that the first roi has a mask roi_has_mask[0] = 1 if cfg.MRCNN.CLS_SPECIFIC_MASK: masks = _expand_to_class_specific_mask_targets(masks, mask_class_labels) # Scale rois_fg and format as (batch_idx, x1, y1, x2, y2) rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # Update blobs dict with Mask R-CNN blobs blobs['mask_rois'] = rois_fg blobs['roi_has_mask_int32'] = roi_has_mask blobs['masks_int32'] = masks
def add_body_uv_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): IsFlipped = roidb['flipped'] M = cfg.BODY_UV_RCNN.HEATMAP_SIZE # polys_gt_inds = np.where(roidb['ignore_UV_body'] == 0)[0] boxes_from_polys = [roidb['boxes'][i, :] for i in polys_gt_inds] input_w = roidb['input_width'] input_h = roidb['input_height'] if not (boxes_from_polys): pass else: boxes_from_polys = np.vstack(boxes_from_polys) boxes_from_polys = np.array(boxes_from_polys) fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = np.zeros(blobs['labels_int32'].shape) if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0)): rois_fg = sampled_boxes[fg_inds] # rois_fg.astype(np.float32, copy=False) boxes_from_polys.astype(np.float32, copy=False) # overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) fg_polys_value = np.max(overlaps_bbfg_bbpolys, axis=1) fg_inds = fg_inds[fg_polys_value > 0.7] all_person_masks = np.zeros((int(input_h), int(input_w)), dtype=np.float32) if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0)): # controle the number of roi if fg_inds.shape[0] > 6: fg_inds = fg_inds[:6] for jj in fg_inds: roi_has_mask[jj] = 1 # Create blobs for densepose supervision. ################################################## The mask All_labels = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) All_Weights = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) ################################################# The points X_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Y_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Ind_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) I_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) U_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) V_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Uv_point_weights = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) ################################################# rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) rois = np.copy(rois_fg) for i in range(rois_fg.shape[0]): # fg_polys_ind = polys_gt_inds[fg_polys_inds[i]] # Ilabel = segm_utils.GetDensePoseMask( roidb['dp_masks'][fg_polys_ind]) # GT_I = np.array(roidb['dp_I'][fg_polys_ind]) GT_U = np.array(roidb['dp_U'][fg_polys_ind]) GT_V = np.array(roidb['dp_V'][fg_polys_ind]) GT_x = np.array(roidb['dp_x'][fg_polys_ind]) GT_y = np.array(roidb['dp_y'][fg_polys_ind]) GT_weights = np.ones(GT_I.shape).astype(np.float32) # ## Do the flipping of the densepose annotation ! if (IsFlipped): GT_I, GT_U, GT_V, GT_x, GT_y, Ilabel = DP.get_symmetric_densepose( GT_I, GT_U, GT_V, GT_x, GT_y, Ilabel) # roi_fg = rois_fg[i] roi_gt = boxes_from_polys[fg_polys_inds[i], :] # x1 = roi_fg[0] x2 = roi_fg[2] y1 = roi_fg[1] y2 = roi_fg[3] # x1_source = roi_gt[0] x2_source = roi_gt[2] y1_source = roi_gt[1] y2_source = roi_gt[3] # x_targets = (np.arange(x1, x2, (x2 - x1) / M) - x1_source) * (256. / (x2_source - x1_source)) y_targets = (np.arange(y1, y2, (y2 - y1) / M) - y1_source) * (256. / (y2_source - y1_source)) # x_targets = x_targets[ 0: M] ## Strangely sometimes it can be M+1, so make sure size is OK! y_targets = y_targets[0:M] # [X_targets, Y_targets] = np.meshgrid(x_targets, y_targets) New_Index = cv2.remap(Ilabel, X_targets.astype(np.float32), Y_targets.astype(np.float32), interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) # All_L = np.zeros(New_Index.shape) All_W = np.ones(New_Index.shape) # All_L = New_Index # gt_length_x = x2_source - x1_source gt_length_y = y2_source - y1_source # GT_y = ((GT_y / 256. * gt_length_y) + y1_source - y1) * (M / (y2 - y1)) GT_x = ((GT_x / 256. * gt_length_x) + x1_source - x1) * (M / (x2 - x1)) # GT_I[GT_y < 0] = 0 GT_I[GT_y > (M - 1)] = 0 GT_I[GT_x < 0] = 0 GT_I[GT_x > (M - 1)] = 0 # points_inside = GT_I > 0 GT_U = GT_U[points_inside] GT_V = GT_V[points_inside] GT_x = GT_x[points_inside] GT_y = GT_y[points_inside] GT_weights = GT_weights[points_inside] GT_I = GT_I[points_inside] # X_points[i, 0:len(GT_x)] = GT_x Y_points[i, 0:len(GT_y)] = GT_y Ind_points[i, 0:len(GT_I)] = i I_points[i, 0:len(GT_I)] = GT_I U_points[i, 0:len(GT_U)] = GT_U V_points[i, 0:len(GT_V)] = GT_V Uv_point_weights[i, 0:len(GT_weights)] = GT_weights # All_labels[i, :] = np.reshape(All_L.astype(np.int32), M**2) All_Weights[i, :] = np.reshape(All_W.astype(np.int32), M**2) ## # proposal based segmentation p_mask = (Ilabel > 0).astype(np.float32) target_roi = roi_gt * im_scale p_mask = cv2.resize(p_mask, (int(target_roi[2] - target_roi[0]), int(target_roi[3] - target_roi[1]))) p_mask = (p_mask > 0.5).astype(np.float32) start_y, start_x = int(target_roi[1]), int(target_roi[0]) end_y, end_x = start_y + p_mask.shape[0], start_x + p_mask.shape[1] # if all_person_masks[start_y:end_y, start_x:end_x].shape[0]!=p_mask.shape[0] or all_person_masks[start_y:end_y, start_x:end_x].shape[1]!=p_mask.shape[1]: # print('shape exception:',all_person_masks[start_y:end_y, start_x:end_x].shape,p_mask.shape) # print('roi:',target_roi) # print(start_y,end_y, start_x,end_x) # print('input image:',all_person_masks.shape) # assert False all_person_masks[start_y:end_y, start_x:end_x] = p_mask else: bg_inds = np.where(blobs['labels_int32'] == 0)[0] # if (len(bg_inds) == 0): rois_fg = sampled_boxes[0].reshape((1, -1)) else: rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) roi_has_mask[0] = 1 # X_points = blob_utils.zeros((1, 196), int32=False) Y_points = blob_utils.zeros((1, 196), int32=False) Ind_points = blob_utils.zeros((1, 196), int32=True) I_points = blob_utils.zeros((1, 196), int32=True) U_points = blob_utils.zeros((1, 196), int32=False) V_points = blob_utils.zeros((1, 196), int32=False) Uv_point_weights = blob_utils.zeros((1, 196), int32=False) # All_labels = -blob_utils.ones((1, M**2), int32=True) * 0 ## zeros All_Weights = -blob_utils.ones((1, M**2), int32=True) * 0 ## zeros # rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # K = cfg.BODY_UV_RCNN.NUM_PATCHES # u_points = np.copy(U_points) v_points = np.copy(V_points) U_points = np.tile(U_points, [1, K + 1]) V_points = np.tile(V_points, [1, K + 1]) Uv_Weight_Points = np.zeros(U_points.shape) # for jjj in xrange(1, K + 1): Uv_Weight_Points[:, jjj * I_points.shape[1]:(jjj + 1) * I_points.shape[1]] = (I_points == jjj).astype( np.float32) # # person masks here person_mask = (All_labels > 0).astype(np.int32) # extra # index_targets = np.zeros_like(person_mask).reshape((-1,M,M)).astype(np.int32) # index_targets_weights = np.zeros_like(index_targets) # u_targets = np.zeros((index_targets.shape[0],25,M,M),dtype=np.float32) # v_targets = np.zeros((index_targets.shape[0], 25, M, M),dtype=np.float32) # uv_weights = np.zeros((index_targets.shape[0], 25, M, M),dtype=np.float32) # for ibatch in range(index_targets.shape[0]): # for i_surface in range(1,K+1): # points_i = I_points[ibatch] == i_surface # if len(points_i)>0: # points_x = np.asarray(X_points[ibatch][points_i], dtype=np.int32).reshape((-1,1)) # points_y = np.asarray(Y_points[ibatch][points_i], dtype=np.int32).reshape((-1,1)) # points_u = u_points[ibatch][points_i].reshape((1, -1)) # points_v = v_points[ibatch][points_i].reshape((1, -1)) # locs = np.hstack([points_x, points_y]) # # for step in [1]: # x_plus_locs = np.copy(points_x) + step # y_plus_locs = np.copy(points_y) + step # x_minus_locs = np.copy(points_x) - step # y_minus_locs = np.copy(points_y) - step # # locs = np.vstack([locs, np.hstack([x_plus_locs, y_plus_locs])]) # locs = np.vstack([locs, np.hstack([x_plus_locs, y_minus_locs])]) # locs = np.vstack([locs, np.hstack([x_minus_locs, y_plus_locs])]) # locs = np.vstack([locs, np.hstack([x_minus_locs, y_minus_locs])]) # # locs[locs < 0] = 0. # locs[locs >= M] = M - 1 # # points_u = np.repeat(points_u, 5, axis=0).reshape((-1)) # points_v = np.repeat(points_v, 5, axis=0).reshape((-1)) # # # index_targets[ibatch][locs[:,1], locs[:, 0]] = i_surface # index_targets_weights[ibatch][locs[:, 1], locs[:, 0]] = 1 # u_targets[ibatch, i_surface][locs[:, 1], locs[:, 0]] = points_u # v_targets[ibatch, i_surface][locs[:, 1], locs[:, 0]] = points_v # uv_weights[ibatch, i_surface][locs[:, 1], locs[:, 0]] = 1. # if random.random() <= 0.5: # _,index_targets[ibatch], v_targets[ibatch], v_targets[ibatch], index_targets_weights[ibatch], uv_weights[ibatch] = expand_dp_targets(All_labels[ibatch].reshape((M,M)), # index_targets[ibatch], v_targets[ibatch], # v_targets[ibatch], # index_targets_weights[ibatch], # uv_weights[ibatch]) # proposal all masks here if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0)): proposal_all_mask = blob_utils.zeros((fg_inds.shape[0], M, M), int32=True) for i in range(rois_fg.shape[0]): roi_fg = rois_fg[i][1:] proposal_mask = all_person_masks[int(roi_fg[1]):int(roi_fg[3]), int(roi_fg[0]):int(roi_fg[2])] proposal_mask = cv2.resize(proposal_mask, (M, M)) proposal_mask = (proposal_mask > 0.5).astype(np.int32) proposal_all_mask[i] = proposal_mask else: proposal_all_mask = -blob_utils.ones( (1, M, M), int32=True) * 0 ## zeros ################ # Update blobs dict with Mask R-CNN blobs ############### # blobs['body_mask_labels'] = person_mask.reshape((-1, M, M)) blobs['body_uv_rois'] = np.array(rois_fg) blobs['roi_has_body_uv_int32'] = np.array(roi_has_mask).astype(np.int32) ## blobs['body_uv_ann_labels'] = np.array(All_labels).astype(np.int32) blobs['body_uv_ann_weights'] = np.array(All_Weights).astype(np.float32) # ########################## blobs['body_uv_X_points'] = X_points.astype(np.float32) blobs['body_uv_Y_points'] = Y_points.astype(np.float32) blobs['body_uv_Ind_points'] = Ind_points.astype(np.float32) blobs['body_uv_I_points'] = I_points.astype(np.float32) blobs['body_uv_U_points'] = U_points.astype( np.float32) #### VERY IMPORTANT : These are switched here : blobs['body_uv_V_points'] = V_points.astype(np.float32) blobs['body_uv_point_weights'] = Uv_Weight_Points.astype(np.float32) ################### # extra # blobs['body_uv_Index_targets'] = index_targets # blobs['body_uv_Index_targets_weights'] = index_targets_weights.astype(np.float32) # blobs['body_uv_U_targets'] = u_targets # blobs['body_uv_V_targets'] = v_targets # blobs['body_uv_weights'] = uv_weights ################ # add by wxh if cfg.BODY_UV_RCNN.USE_CLS_EMBS: fg_embs, bg_embs, fg_weights, bg_weights = masks_to_embs( All_labels.reshape((-1, M, M))) # print('fg',fg_embs.max(), fg_embs.min()) # print('bg',bg_embs.max(), bg_embs.min()) fg_norms = np.sum(fg_embs, axis=(1, 2)) fg_norms[fg_norms != 0] = 56. * 56. / fg_norms[fg_norms != 0] bg_norms = np.sum(bg_embs, axis=(1, 2)) bg_norms[bg_norms != 0] = 56. * 56. / bg_norms[bg_norms != 0] blobs['fg_mask'] = np.repeat(np.reshape(fg_embs, (-1, 1, M, M)), 2, axis=1) blobs['bg_mask'] = np.repeat(np.reshape(bg_embs, (-1, 1, M, M)), 2, axis=1) blobs['fg_norm'] = np.repeat(np.reshape(fg_norms, (-1, 1)), 2, axis=1) blobs['bg_norm'] = np.repeat(np.reshape(bg_norms, (-1, 1)), 2, axis=1) blobs['mask_emb_fg_labels'] = np.ones((fg_embs.shape[0], 1), dtype=np.int32) blobs['mask_emb_bg_labels'] = np.zeros((bg_embs.shape[0], 1), dtype=np.int32) blobs['mask_emb_weights'] = np.vstack([fg_weights, bg_weights]).reshape( (-1, 1)).astype(np.float32) if cfg.BODY_UV_RCNN.USE_BOX_ALL_MASKS: blobs['body_masks_wrt_box'] = proposal_all_mask
def add_body_uv_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): IsFlipped = roidb['flipped'] M = cfg.BODY_UV_RCNN.HEATMAP_SIZE # polys_gt_inds = np.where(roidb['ignore_UV_body'] == 0)[0] boxes_from_polys = [roidb['boxes'][i,:] for i in polys_gt_inds] if not(boxes_from_polys): pass else: boxes_from_polys = np.vstack(boxes_from_polys) boxes_from_polys = np.array(boxes_from_polys) fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = np.zeros( blobs['labels_int32'].shape ) if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0) ): rois_fg = sampled_boxes[fg_inds] # rois_fg.astype(np.float32, copy=False) boxes_from_polys.astype(np.float32, copy=False) # overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) fg_polys_value = np.max(overlaps_bbfg_bbpolys, axis=1) fg_inds = fg_inds[fg_polys_value>0.7] if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0) ): for jj in fg_inds: roi_has_mask[jj] = 1 # Create blobs for densepose supervision. ################################################## The mask All_labels = blob_utils.zeros((fg_inds.shape[0], M ** 2), int32=True) All_Weights = blob_utils.zeros((fg_inds.shape[0], M ** 2), int32=True) ################################################# The points X_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Y_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Ind_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) I_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) U_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) V_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Uv_point_weights = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) ################################################# rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) for i in range(rois_fg.shape[0]): # fg_polys_ind = polys_gt_inds[ fg_polys_inds[i] ] # Ilabel = segm_utils.GetDensePoseMask( roidb['dp_masks'][ fg_polys_ind ] ) # GT_I = np.array(roidb['dp_I'][ fg_polys_ind ]) GT_U = np.array(roidb['dp_U'][ fg_polys_ind ]) GT_V = np.array(roidb['dp_V'][ fg_polys_ind ]) GT_x = np.array(roidb['dp_x'][ fg_polys_ind ]) GT_y = np.array(roidb['dp_y'][ fg_polys_ind ]) GT_weights = np.ones(GT_I.shape).astype(np.float32) # ## Do the flipping of the densepose annotation ! if(IsFlipped): GT_I,GT_U,GT_V,GT_x,GT_y,Ilabel = DP.get_symmetric_densepose(GT_I,GT_U,GT_V,GT_x,GT_y,Ilabel) # roi_fg = rois_fg[i] roi_gt = boxes_from_polys[fg_polys_inds[i],:] # x1 = roi_fg[0] ; x2 = roi_fg[2] y1 = roi_fg[1] ; y2 = roi_fg[3] # x1_source = roi_gt[0]; x2_source = roi_gt[2] y1_source = roi_gt[1]; y2_source = roi_gt[3] # x_targets = ( np.arange(x1,x2, (x2 - x1)/M ) - x1_source ) * ( 256. / (x2_source-x1_source) ) y_targets = ( np.arange(y1,y2, (y2 - y1)/M ) - y1_source ) * ( 256. / (y2_source-y1_source) ) # x_targets = x_targets[0:M] ## Strangely sometimes it can be M+1, so make sure size is OK! y_targets = y_targets[0:M] # [X_targets,Y_targets] = np.meshgrid( x_targets, y_targets ) New_Index = cv2.remap(Ilabel,X_targets.astype(np.float32), Y_targets.astype(np.float32), interpolation=cv2.INTER_NEAREST, borderMode= cv2.BORDER_CONSTANT, borderValue=(0)) # All_L = np.zeros(New_Index.shape) All_W = np.ones(New_Index.shape) # All_L = New_Index # gt_length_x = x2_source - x1_source gt_length_y = y2_source - y1_source # GT_y = (( GT_y / 256. * gt_length_y ) + y1_source - y1 ) * ( M / ( y2 - y1 ) ) GT_x = (( GT_x / 256. * gt_length_x ) + x1_source - x1 ) * ( M / ( x2 - x1 ) ) # GT_I[GT_y<0] = 0 GT_I[GT_y>(M-1)] = 0 GT_I[GT_x<0] = 0 GT_I[GT_x>(M-1)] = 0 # points_inside = GT_I>0 GT_U = GT_U[points_inside] GT_V = GT_V[points_inside] GT_x = GT_x[points_inside] GT_y = GT_y[points_inside] GT_weights = GT_weights[points_inside] GT_I = GT_I[points_inside] # X_points[i, 0:len(GT_x)] = GT_x Y_points[i, 0:len(GT_y)] = GT_y Ind_points[i, 0:len(GT_I)] = i I_points[i, 0:len(GT_I)] = GT_I U_points[i, 0:len(GT_U)] = GT_U V_points[i, 0:len(GT_V)] = GT_V Uv_point_weights[i, 0:len(GT_weights)] = GT_weights # All_labels[i, :] = np.reshape(All_L.astype(np.int32), M ** 2) All_Weights[i, :] = np.reshape(All_W.astype(np.int32), M ** 2) ## else: bg_inds = np.where(blobs['labels_int32'] == 0)[0] # if(len(bg_inds)==0): rois_fg = sampled_boxes[0].reshape((1, -1)) else: rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) roi_has_mask[0] = 1 # X_points = blob_utils.zeros((1, 196), int32=False) Y_points = blob_utils.zeros((1, 196), int32=False) Ind_points = blob_utils.zeros((1, 196), int32=True) I_points = blob_utils.zeros((1,196), int32=True) U_points = blob_utils.zeros((1, 196), int32=False) V_points = blob_utils.zeros((1, 196), int32=False) Uv_point_weights = blob_utils.zeros((1, 196), int32=False) # All_labels = -blob_utils.ones((1, M ** 2), int32=True) * 0 ## zeros All_Weights = -blob_utils.ones((1, M ** 2), int32=True) * 0 ## zeros # rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # K = cfg.BODY_UV_RCNN.NUM_PATCHES # U_points = np.tile( U_points , [1,K+1] ) V_points = np.tile( V_points , [1,K+1] ) Uv_Weight_Points = np.zeros(U_points.shape) # for jjj in xrange(1,K+1): Uv_Weight_Points[ : , jjj * I_points.shape[1] : (jjj+1) * I_points.shape[1] ] = ( I_points == jjj ).astype(np.float32) # ################ # Update blobs dict with Mask R-CNN blobs ############### # blobs['body_uv_rois'] = np.array(rois_fg) blobs['roi_has_body_uv_int32'] = np.array(roi_has_mask).astype(np.int32) ## blobs['body_uv_ann_labels'] = np.array(All_labels).astype(np.int32) blobs['body_uv_ann_weights'] = np.array(All_Weights).astype(np.float32) # ########################## blobs['body_uv_X_points'] = X_points.astype(np.float32) blobs['body_uv_Y_points'] = Y_points.astype(np.float32) blobs['body_uv_Ind_points'] = Ind_points.astype(np.float32) blobs['body_uv_I_points'] = I_points.astype(np.float32) blobs['body_uv_U_points'] = U_points.astype(np.float32) #### VERY IMPORTANT : These are switched here : blobs['body_uv_V_points'] = V_points.astype(np.float32) blobs['body_uv_point_weights'] = Uv_Weight_Points.astype(np.float32)
def add_body_uv_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): IsFlipped = roidb['flipped'] M = cfg.BODY_UV_RCNN.HEATMAP_SIZE # polys_gt_inds = np.where(roidb['ignore_UV_body'] == 0)[0] boxes_from_polys = [roidb['boxes'][i, :] for i in polys_gt_inds] if not (boxes_from_polys): pass else: boxes_from_polys = np.vstack(boxes_from_polys) boxes_from_polys = np.array(boxes_from_polys) fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_mask = np.zeros(blobs['labels_int32'].shape) if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0)): rois_fg = sampled_boxes[fg_inds] # rois_fg.astype(np.float32, copy=False) boxes_from_polys.astype(np.float32, copy=False) # overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) fg_polys_value = np.max(overlaps_bbfg_bbpolys, axis=1) fg_inds = fg_inds[fg_polys_value > 0.7] if (bool(boxes_from_polys.any()) & (fg_inds.shape[0] > 0)): for jj in fg_inds: roi_has_mask[jj] = 1 # Create blobs for densepose supervision. ################################################## The mask All_labels = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) All_Weights = blob_utils.zeros((fg_inds.shape[0], M**2), int32=True) ################################################# The points X_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Y_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Ind_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) I_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) U_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) V_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) Uv_point_weights = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) ################################################# rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) for i in range(rois_fg.shape[0]): # fg_polys_ind = polys_gt_inds[fg_polys_inds[i]] # Ilabel = segm_utils.GetDensePoseMask( roidb['dp_masks'][fg_polys_ind]) # GT_I = np.array(roidb['dp_I'][fg_polys_ind]) GT_U = np.array(roidb['dp_U'][fg_polys_ind]) GT_V = np.array(roidb['dp_V'][fg_polys_ind]) GT_x = np.array(roidb['dp_x'][fg_polys_ind]) GT_y = np.array(roidb['dp_y'][fg_polys_ind]) GT_weights = np.ones(GT_I.shape).astype(np.float32) # ## Do the flipping of the densepose annotation ! if (IsFlipped): GT_I, GT_U, GT_V, GT_x, GT_y, Ilabel = DP.get_symmetric_densepose( GT_I, GT_U, GT_V, GT_x, GT_y, Ilabel) # roi_fg = rois_fg[i] roi_gt = boxes_from_polys[fg_polys_inds[i], :] # x1 = roi_fg[0] x2 = roi_fg[2] y1 = roi_fg[1] y2 = roi_fg[3] # x1_source = roi_gt[0] x2_source = roi_gt[2] y1_source = roi_gt[1] y2_source = roi_gt[3] # x_targets = (np.arange(x1, x2, (x2 - x1) / M) - x1_source) * (256. / (x2_source - x1_source)) y_targets = (np.arange(y1, y2, (y2 - y1) / M) - y1_source) * (256. / (y2_source - y1_source)) # x_targets = x_targets[ 0: M] ## Strangely sometimes it can be M+1, so make sure size is OK! y_targets = y_targets[0:M] # [X_targets, Y_targets] = np.meshgrid(x_targets, y_targets) New_Index = cv2.remap(Ilabel, X_targets.astype(np.float32), Y_targets.astype(np.float32), interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) # All_L = np.zeros(New_Index.shape) All_W = np.ones(New_Index.shape) # All_L = New_Index # gt_length_x = x2_source - x1_source gt_length_y = y2_source - y1_source # GT_y = ((GT_y / 256. * gt_length_y) + y1_source - y1) * (M / (y2 - y1)) GT_x = ((GT_x / 256. * gt_length_x) + x1_source - x1) * (M / (x2 - x1)) # GT_I[GT_y < 0] = 0 GT_I[GT_y > (M - 1)] = 0 GT_I[GT_x < 0] = 0 GT_I[GT_x > (M - 1)] = 0 # points_inside = GT_I > 0 GT_U = GT_U[points_inside] GT_V = GT_V[points_inside] GT_x = GT_x[points_inside] GT_y = GT_y[points_inside] GT_weights = GT_weights[points_inside] GT_I = GT_I[points_inside] # X_points[i, 0:len(GT_x)] = GT_x Y_points[i, 0:len(GT_y)] = GT_y Ind_points[i, 0:len(GT_I)] = i I_points[i, 0:len(GT_I)] = GT_I U_points[i, 0:len(GT_U)] = GT_U V_points[i, 0:len(GT_V)] = GT_V Uv_point_weights[i, 0:len(GT_weights)] = GT_weights # All_labels[i, :] = np.reshape(All_L.astype(np.int32), M**2) All_Weights[i, :] = np.reshape(All_W.astype(np.int32), M**2) ## else: bg_inds = np.where(blobs['labels_int32'] == 0)[0] # if (len(bg_inds) == 0): rois_fg = sampled_boxes[0].reshape((1, -1)) else: rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) roi_has_mask[0] = 1 # X_points = blob_utils.zeros((1, 196), int32=False) Y_points = blob_utils.zeros((1, 196), int32=False) Ind_points = blob_utils.zeros((1, 196), int32=True) I_points = blob_utils.zeros((1, 196), int32=True) U_points = blob_utils.zeros((1, 196), int32=False) V_points = blob_utils.zeros((1, 196), int32=False) Uv_point_weights = blob_utils.zeros((1, 196), int32=False) # All_labels = -blob_utils.ones((1, M**2), int32=True) * 0 ## zeros All_Weights = -blob_utils.ones((1, M**2), int32=True) * 0 ## zeros # rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # K = cfg.BODY_UV_RCNN.NUM_PATCHES # U_points = np.tile(U_points, [1, K + 1]) V_points = np.tile(V_points, [1, K + 1]) Uv_Weight_Points = np.zeros(U_points.shape) # for jjj in xrange(1, K + 1): Uv_Weight_Points[:, jjj * I_points.shape[1]:(jjj + 1) * I_points.shape[1]] = (I_points == jjj).astype( np.float32) # ################ # Update blobs dict with Mask R-CNN blobs ############### # blobs['body_uv_rois'] = np.array(rois_fg) blobs['roi_has_body_uv_int32'] = np.array(roi_has_mask).astype(np.int32) ## blobs['body_uv_ann_labels'] = np.array(All_labels).astype(np.int32) blobs['body_uv_ann_weights'] = np.array(All_Weights).astype(np.float32) # ########################## blobs['body_uv_X_points'] = X_points.astype(np.float32) blobs['body_uv_Y_points'] = Y_points.astype(np.float32) blobs['body_uv_Ind_points'] = Ind_points.astype(np.float32) blobs['body_uv_I_points'] = I_points.astype(np.float32) blobs['body_uv_U_points'] = U_points.astype( np.float32) #### VERY IMPORTANT : These are switched here : blobs['body_uv_V_points'] = V_points.astype(np.float32) blobs['body_uv_point_weights'] = Uv_Weight_Points.astype(np.float32)
def add_body_uv_rcnn_blobs(blobs, sampled_boxes, roidb, im_scale, batch_idx): """Add DensePose specific blobs to the given inputs blobs dictionary.""" M = cfg.BODY_UV_RCNN.HEATMAP_SIZE # Prepare the body UV targets by associating one gt box which contains # body UV annotations to each training roi that has a fg class label. polys_gt_inds = np.where(roidb['ignore_UV_body'] == 0)[0] boxes_from_polys = roidb['boxes'][polys_gt_inds] # Select foreground RoIs fg_inds = np.where(blobs['labels_int32'] > 0)[0] roi_has_body_uv = np.zeros_like(blobs['labels_int32'], dtype=np.int32) if ((boxes_from_polys.shape[0] > 0) & (fg_inds.shape[0] > 0)): # Find overlap between all foreground RoIs and the gt bounding boxes # containing each body UV annotaion. rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = box_utils.bbox_overlaps( rois_fg.astype(np.float32, copy=False), boxes_from_polys.astype(np.float32, copy=False)) # Select foreground RoIs as those with > 0.7 overlap fg_polys_value = np.max(overlaps_bbfg_bbpolys, axis=1) fg_inds = fg_inds[fg_polys_value > 0.7] if ((boxes_from_polys.shape[0] > 0) & (fg_inds.shape[0] > 0)): roi_has_body_uv[fg_inds] = 1 # Create body UV blobs # Dense masks, each mask for a given fg roi is of size M x M. part_inds = blob_utils.zeros((fg_inds.shape[0], M, M), int32=True) # Weights assigned to each target in `part_inds`. By default, all 1's. # part_inds_weights = blob_utils.zeros((fg_inds.shape[0], M, M), int32=True) part_inds_weights = blob_utils.ones((fg_inds.shape[0], M, M), int32=False) # 2D spatial coordinates (on the image). Shape is (#fg_rois, 2) in format # (x, y). coords_xy = blob_utils.zeros((fg_inds.shape[0], 196, 2), int32=False) # 24 patch indices plus a background class I_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=True) # UV coordinates in each patch U_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) V_points = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) # Uv_point_weights = blob_utils.zeros((fg_inds.shape[0], 196), int32=False) rois_fg = sampled_boxes[fg_inds] overlaps_bbfg_bbpolys = overlaps_bbfg_bbpolys[fg_inds] # Map from each fg roi to the index of the gt box with highest overlap fg_polys_inds = np.argmax(overlaps_bbfg_bbpolys, axis=1) # Add body UV targets for each fg roi for i in range(rois_fg.shape[0]): fg_polys_ind = fg_polys_inds[i] polys_gt_ind = polys_gt_inds[fg_polys_ind] # RLE encoded dense masks which are of size 256 x 256. # Map all part masks to 14 labels (i.e., indices of semantic body parts). dp_masks = dp_utils.GetDensePoseMask( roidb['dp_masks'][polys_gt_ind], cfg.BODY_UV_RCNN.NUM_SEMANTIC_PARTS) # Surface patch indices of collected points dp_I = np.array(roidb['dp_I'][polys_gt_ind], dtype=np.int32) # UV coordinates of collected points dp_U = np.array(roidb['dp_U'][polys_gt_ind], dtype=np.float32) dp_V = np.array(roidb['dp_V'][polys_gt_ind], dtype=np.float32) # dp_UV_weights = np.ones_like(dp_I).astype(np.float32) # Spatial coordinates on the image which are scaled such that the bbox # size is 256 x 256. dp_x = np.array(roidb['dp_x'][polys_gt_ind], dtype=np.float32) dp_y = np.array(roidb['dp_y'][polys_gt_ind], dtype=np.float32) # Do the flipping of the densepose annotation if roidb['flipped']: dp_I, dp_U, dp_V, dp_x, dp_y, dp_masks = DP.get_symmetric_densepose( dp_I, dp_U, dp_V, dp_x, dp_y, dp_masks) roi_fg = rois_fg[i] gt_box = boxes_from_polys[fg_polys_ind] fg_x1, fg_y1, fg_x2, fg_y2 = roi_fg[0:4] gt_x1, gt_y1, gt_x2, gt_y2 = gt_box[0:4] fg_width = fg_x2 - fg_x1 fg_height = fg_y2 - fg_y1 gt_width = gt_x2 - gt_x1 gt_height = gt_y2 - gt_y1 fg_scale_w = float(M) / fg_width fg_scale_h = float(M) / fg_height gt_scale_w = 256. / gt_width gt_scale_h = 256. / gt_height # Sample M points evenly within the fg roi and scale the relative coordinates # (to associated gt box) such that the bounding box size is 256 x 256. x_targets = (np.arange(fg_x1, fg_x2, fg_width / M) - gt_x1) * gt_scale_w y_targets = (np.arange(fg_y1, fg_y2, fg_height / M) - gt_y1) * gt_scale_h # Construct 2D coordiante matrices x_targets, y_targets = np.meshgrid(x_targets[:M], y_targets[:M]) ## Another implementation option (which results in similar performance) # x_targets = (np.linspace(fg_x1, fg_x2, M, endpoint=True, dtype=np.float32) - gt_x1) * gt_scale_w # y_targets = (np.linspace(fg_y1, fg_y2, M, endpoint=True, dtype=np.float32) - gt_y1) * gt_scale_h # x_targets = (np.linspace(fg_x1, fg_x2, M, endpoint=False) - gt_x1) * gt_scale_w # y_targets = (np.linspace(fg_y1, fg_y2, M, endpoint=False) - gt_y1) * gt_scale_h # x_targets, y_targets = np.meshgrid(x_targets, y_targets) # Map dense masks of size 256 x 256 to target heatmap of size M x M. part_inds[i] = cv2.remap(dp_masks, x_targets.astype(np.float32), y_targets.astype(np.float32), interpolation=cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT, borderValue=(0)) # Scale annotated spatial coordinates from bbox of size 256 x 256 to target # heatmap of size M x M. dp_x = (dp_x / gt_scale_w + gt_x1 - fg_x1) * fg_scale_w dp_y = (dp_y / gt_scale_h + gt_y1 - fg_y1) * fg_scale_h # Set patch index of points outside the heatmap as 0 (background). dp_I[dp_x < 0] = 0 dp_I[dp_x > (M - 1)] = 0 dp_I[dp_y < 0] = 0 dp_I[dp_y > (M - 1)] = 0 # Get body UV annotations of points inside the heatmap. points_inside = dp_I > 0 dp_x = dp_x[points_inside] dp_y = dp_y[points_inside] dp_I = dp_I[points_inside] dp_U = dp_U[points_inside] dp_V = dp_V[points_inside] # dp_UV_weights = dp_UV_weights[points_inside] # Update body UV blobs num_dp_points = len(dp_I) # coords_xy[i, 0:num_dp_points, 0] = i # fg_roi index coords_xy[i, 0:num_dp_points, 0] = dp_x coords_xy[i, 0:num_dp_points, 1] = dp_y I_points[i, 0:num_dp_points] = dp_I.astype(np.int32) U_points[i, 0:num_dp_points] = dp_U V_points[i, 0:num_dp_points] = dp_V # Uv_point_weights[i, 0:len(dp_UV_weights)] = dp_UV_weights else: # If there are no fg rois # The network cannot handle empty blobs, so we must provide a blob. # We simply take the first bg roi, give it an all 0's body UV annotations # and label it with class zero (bg). bg_inds = np.where(blobs['labels_int32'] == 0)[0] # `rois_fg` is actually one background roi, but that's ok because ... if len(bg_inds) == 0: rois_fg = sampled_boxes[0].reshape((1, -1)) else: rois_fg = sampled_boxes[bg_inds[0]].reshape((1, -1)) # Mark that the first roi has body UV annotation roi_has_body_uv[0] = 1 # We give it all 0's blobs part_inds = blob_utils.zeros((1, M, M), int32=True) part_inds_weights = blob_utils.zeros((1, M, M), int32=False) coords_xy = blob_utils.zeros((1, 196, 2), int32=False) I_points = blob_utils.zeros((1, 196), int32=True) U_points = blob_utils.zeros((1, 196), int32=False) V_points = blob_utils.zeros((1, 196), int32=False) # Uv_point_weights = blob_utils.zeros((1, 196), int32=False) # Scale rois_fg and format as (batch_idx, x1, y1, x2, y2) rois_fg *= im_scale repeated_batch_idx = batch_idx * blob_utils.ones((rois_fg.shape[0], 1)) rois_fg = np.hstack((repeated_batch_idx, rois_fg)) # Create body UV blobs for all patches (including background) K = cfg.BODY_UV_RCNN.NUM_PATCHES + 1 # Construct U/V_points blobs for all patches by repeating it #num_patches times. # Shape: (#rois, 196, K) U_points = np.repeat(U_points[:, :, np.newaxis], K, axis=-1) V_points = np.repeat(V_points[:, :, np.newaxis], K, axis=-1) uv_point_weights = np.zeros_like(U_points) # Set binary weights for UV targets in each patch for i in np.arange(1, K): uv_point_weights[:, :, i] = (I_points == i).astype(np.float32) # Update blobs dict with body UV blobs blobs['body_uv_rois'] = rois_fg blobs['roi_has_body_uv_int32'] = roi_has_body_uv # shape: (#rois,) blobs['body_uv_parts'] = part_inds # shape: (#rois, M, M) blobs['body_uv_parts_weights'] = part_inds_weights blobs['body_uv_coords_xy'] = coords_xy.reshape( -1, 2) # shape: (#rois * 196, 2) blobs['body_uv_I_points'] = I_points.reshape(-1, 1) # shape: (#rois * 196, 1) blobs['body_uv_U_points'] = U_points # shape: (#rois, 196, K) blobs['body_uv_V_points'] = V_points blobs['body_uv_point_weights'] = uv_point_weights