def vis_detections(lidar_bv, image, calib, bbox_pred_cnr, rpn_data, rpn_rois, rcnn_roi, scores, gt_boxes_3d): import matplotlib.pyplot as plt from utils.transform import lidar_3d_to_corners, corners_to_bv from fast_rcnn.bbox_transform import bbox_transform_inv_cnr from utils.draw import show_lidar_corners, show_image_boxes, scale_to_255 from utils.cython_nms import nms, nms_new image = image.reshape((image.shape[1], image.shape[2], image.shape[3])) image += cfg.PIXEL_MEANS image = image.astype(np.uint8, copy=False) lidar_bv = lidar_bv.reshape( (lidar_bv.shape[1], lidar_bv.shape[2], lidar_bv.shape[3]))[:, :, 8] # visualize anchor_target_layer output rpn_anchors_3d = rpn_data[3][:, 1:7] rpn_bv = rpn_data[2][:, 1:5] # rpn_label = rpn_data[0] # print rpn_label.shape # print rpn_label[rpn_label==1] rpn_boxes_cnr = lidar_3d_to_corners(rpn_anchors_3d) img = show_lidar_corners(image, rpn_boxes_cnr, calib) img_bv = show_image_boxes(scale_to_255(lidar_bv, min=0, max=2), rpn_bv) print img.shape # plt.ion() plt.title('anchor target layer before regression') plt.subplot(211) plt.imshow(img_bv) plt.subplot(212) plt.imshow(img) plt.show() # visualize proposal_layer output boxes_3d = rpn_rois[2][:, 1:7] boxes_bv = rpn_rois[0][:, 0:5] boxes_img = rpn_rois[1][:, 0:5] # keep = nms(boxes_img, cfg.TEST.NMS) # boxes_img = boxes_img[keep] # boxes_3d = boxes_3d[keep] # boxes_cnr = lidar_3d_to_corners(boxes_3d[:100]) print boxes_3d.shape print boxes_bv.shape # image_cnr = show_lidar_corners(image, boxes_cnr, calib) image_bv = show_image_boxes(lidar_bv, boxes_bv[:, 1:5]) image_img = show_image_boxes(image, boxes_img[:, 1:5]) plt.title('proposal_layer ') plt.subplot(211) plt.imshow(image_bv) plt.subplot(212) plt.imshow(image_img) plt.show()
def proposal_layer_3d(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, calib, cfg_key, _feat_stride=[ 8, ], anchor_scales=[1.0, 1.0]): # Algorithm: # # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) #layer_params = yaml.load(self.param_str_) _anchors = generate_anchors_bv() # _anchors = generate_anchors(scales=np.array(anchor_scales)) _num_anchors = _anchors.shape[0] #print "aaaaaaa",_anchors.shape (4,4) #print "bbbbbbb",im_info (601,601,1) #print "ccccccc", calib.shape (4,12) im_info = im_info[0] assert rpn_cls_prob_reshape.shape[0] == 1, \ 'Only single item batches are supported' # cfg_key = str(self.phase) # either 'TRAIN' or 'TEST' pre_nms_topN = cfg[cfg_key].RPN_PRE_NMS_TOP_N post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N nms_thresh = cfg[cfg_key].RPN_NMS_THRESH min_size = cfg[cfg_key].RPN_MIN_SIZE # the first set of _num_anchors channels are bg probs # the second set are the fg probs, which we want # print rpn_cls_prob_reshape.shape height, width = rpn_cls_prob_reshape.shape[1:3] # scores = rpn_cls_prob_reshape[:, _num_anchors:, :, :] scores = np.reshape( np.reshape(rpn_cls_prob_reshape, [1, height, width, _num_anchors, 2])[:, :, :, :, 1], [1, height, width, _num_anchors]) bbox_deltas = rpn_bbox_pred if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) # 1. Generate proposals from bbox deltas and shifted anchors if DEBUG: print 'score map size: {}'.format(scores.shape) # Enumerate all shifts shift_x = np.arange(0, width) * _feat_stride shift_y = np.arange(0, height) * _feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = _num_anchors K = shifts.shape[0] anchors = _anchors.reshape((1, A, 4)) + \ shifts.reshape((1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order # bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 6)) bbox_deltas = bbox_deltas.reshape((-1, 6)) # print "bbox_deltas",bbox_deltas.shape # print anchors.shape # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) # scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) scores = scores.reshape((-1, 1)) # print np.sort(scores.ravel())[-30:] # convert anchors bv to anchors_3d anchors_3d = bv_anchor_to_lidar(anchors) # Convert anchors into proposals via bbox transformations proposals_3d = bbox_transform_inv_3d(anchors_3d, bbox_deltas) # convert back to lidar_bv proposals_bv = lidar_3d_to_bv(proposals_3d) lidar_corners = lidar_3d_to_corners(proposals_3d) proposals_img = lidar_cnr_to_img(lidar_corners, calib[3], calib[2], calib[0]) if DEBUG: # print "bbox_deltas: ", bbox_deltas[:10] # print "proposals number: ", proposals_3d[:10] print "proposals_bv shape: ", proposals_bv.shape print "proposals_3d shape: ", proposals_3d.shape print "proposals_img shape:", proposals_img.shape # 2. clip predicted boxes to image proposals_bv = clip_boxes(proposals_bv, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = _filter_boxes(proposals_bv, min_size * im_info[2]) proposals_bv = proposals_bv[keep, :] proposals_3d = proposals_3d[keep, :] proposals_img = proposals_img[keep, :] scores = scores[keep] # TODO: pass real image_info keep = _filter_img_boxes(proposals_img, [375, 1242]) proposals_bv = proposals_bv[keep, :] proposals_3d = proposals_3d[keep, :] proposals_img = proposals_img[keep, :] scores = scores[keep] if DEBUG: print "proposals after clip" print "proposals_bv shape: ", proposals_bv.shape print "proposals_3d shape: ", proposals_3d.shape print "proposals_img shape: ", proposals_img.shape # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals_bv = proposals_bv[order, :] proposals_3d = proposals_3d[order, :] proposals_img = proposals_img[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) keep = nms(np.hstack((proposals_bv, scores)), nms_thresh) if post_nms_topN > 0: keep = keep[:post_nms_topN] proposals_bv = proposals_bv[keep, :] proposals_3d = proposals_3d[keep, :] proposals_img = proposals_img[keep, :] scores = scores[keep] if DEBUG: print "proposals after nms" print "proposals_bv shape: ", proposals_bv.shape print "proposals_3d shape: ", proposals_3d.shape # Output rois blob # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals_bv.shape[0], 1), dtype=np.float32) blob_bv = np.hstack((batch_inds, proposals_bv.astype(np.float32, copy=False))) blob_img = np.hstack( (batch_inds, proposals_img.astype(np.float32, copy=False))) blob_3d = np.hstack((batch_inds, proposals_3d.astype(np.float32, copy=False))) if DEBUG: print "blob shape ====================:" print blob_bv.shape print blob_img.shape # print '3d', blob_3d[:10] # print lidar_corners[:10] # print 'bv', blob_bv[:10] # print 'img', blob_img[:10] return blob_bv, blob_img, blob_3d
def box_detect(sess, net, im, bv, calib, boxes=None): """Detect object classes in an lidar bv given object proposals. Arguments: net (caffe.Net): Fast R-CNN network to use bv (ndarray): lidar bv to test boxes (ndarray): R x 4 array of object proposals Returns: scores (ndarray): R x K array of object class scores (K includes background as object category 0) boxes (ndarray): R x (4*K) array of predicted bounding boxes """ im_blob = im - cfg.PIXEL_MEANS lidar_bv_blob = bv im_blob = im_blob.reshape( (1, im_blob.shape[0], im_blob.shape[1], im_blob.shape[2])) lidar_bv_blob = lidar_bv_blob.reshape( (1, lidar_bv_blob.shape[0], lidar_bv_blob.shape[1], lidar_bv_blob.shape[2])) blobs = {'image_data': im_blob, 'lidar_bv_data': lidar_bv_blob} im_scales = [1] blobs['calib'] = calib bv_blob = blobs['lidar_bv_data'] blobs['im_info'] = np.array( [[bv_blob.shape[1], bv_blob.shape[2], im_scales[0]]], dtype=np.float32) # forward pass feed_dict = { net.lidar_bv_data: blobs['lidar_bv_data'], net.image_data: blobs['image_data'], net.im_info: blobs['im_info'], net.calib: blobs['calib'], net.keep_prob: 1.0 } conv5_3, deconv, rpn_cls_prob, rpn_cls_prob_reshape, rpn_cls_score_reshape, rpn_cls_score, cls_score, cls_prob, bbox_pred_cnr, rois = sess.run( [ net.get_output('conv5_3_2'), net.get_output('conv5_3'), net.get_output('rpn_cls_prob'), net.get_output('rpn_cls_prob_reshape'), net.get_output('rpn_cls_score_reshape'), net.get_output('rpn_cls_score'), net.get_output('cls_score'), net.get_output('cls_prob'), net.get_output('bbox_pred'), net.get_output('rois') ], feed_dict=feed_dict) scores = cls_prob # plot featuremaps # print conv5_3.shape # # print deconv1.shape # activation = conv5_3 # # featuremaps = activation.shape[3] # featuremaps = 48 # plt.figure(1, figsize=(15,15)) # for featuremap in range(featuremaps): # plt.subplot(6,8, featuremap+1) # sets the number of feature maps to show on each row and column # # plt.title('FeatureMap ' + str(featuremap)) # displays the feature map number # plt.axis('off') # plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", cmap="jet") # plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.01, hspace=0.01) # plt.tight_layout(pad=0.1, h_pad=0.001, w_pad=0.001) # plt.show() # activation = deconv # print deconv.shape # # featuremaps = activation.shape[3] # featuremaps = 48 # plt.figure(1, figsize=(15,15)) # for featuremap in range(featuremaps): # plt.subplot(6,8, featuremap+1) # sets the number of feature maps to show on each row and column # # plt.title('FeatureMap ' + str(featuremap)) # displays the feature map number # plt.axis('off') # plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", cmap="jet") # plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=0.01, hspace=0.01) # plt.tight_layout(pad=0.1, h_pad=0.001, w_pad=0.001) # plt.show() # print cls_score[np.where(cls_score[:,1] > 0)] # plt.hist(cls_score[:,1], bins=25) # plt.show() # plt.hist(scores[:,1], bins=25) # plt.show() assert len(im_scales) == 1, "Only single-image batch implemented" boxes_3d = rois[2][:, 1:7] # Apply bounding-box regression deltas box_deltas = bbox_pred_cnr boxes_cnr = lidar_3d_to_corners(boxes_3d) # img_boxes = lidar_cnr_to_img(boxes_cnr, calib[3], calib[2], calib[0]) # img = show_image_boxes(im, img_boxes) # plt.imshow(img) # plt.show() # !! Important # ! Not apply corner regression pred_boxes_cnr = np.hstack((boxes_cnr, boxes_cnr)) # apply corner regression pred_boxes_cnr_r = bbox_transform_inv_cnr(boxes_cnr, box_deltas) # preject corners to lidar_bv pred_boxes_bv = corners_to_bv(pred_boxes_cnr) return scores, pred_boxes_bv, pred_boxes_cnr, pred_boxes_cnr_r
def proposal_layer_3d_debug(rpn_cls_prob_reshape,rpn_bbox_pred,im_info,calib,cfg_in, _feat_stride = [8,], anchor_scales=[1.0, 1.0],debug_state=True): #copy part of the code from proposal_layer_3d for debug _anchors = generate_anchors_bv() # _anchors = generate_anchors(scales=np.array(anchor_scales)) _num_anchors = _anchors.shape[0] im_info = im_info[0] assert rpn_cls_prob_reshape.shape[0] == 1, \ 'Only single item batches are supported' # cfg_key = str(self.phase) # either 'TRAIN' or 'TEST' # the first set of _num_anchors channels are bg probs # the second set are the fg probs, which we want # print rpn_cls_prob_reshape.shape height, width = rpn_cls_prob_reshape.shape[1:3] # scores = rpn_cls_prob_reshape[:, _num_anchors:, :, :] scores = np.reshape(np.reshape(rpn_cls_prob_reshape, [1, height, width, _num_anchors, 2])[:,:,:,:,1],[1, height, width, _num_anchors]) bbox_deltas = rpn_bbox_pred if debug_state: print ('im_size: ({}, {})'.format(im_info[0], im_info[1])) print ('scale: {}'.format(im_info[2])) if debug_state: print ('score map size: {}'.format(scores.shape)) # Enumerate all shifts shift_x = np.arange(0, width) * _feat_stride shift_y = np.arange(0, height) * _feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = _num_anchors K = shifts.shape[0] anchors = _anchors.reshape((1, A, 4)) + \ shifts.reshape((1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) bbox_deltas = bbox_deltas.reshape((-1, 6)) scores = scores.reshape((-1, 1)) # convert anchors bv to anchors_3d anchors_3d = bv_anchor_to_lidar(anchors) # Convert anchors into proposals via bbox transformations proposals_3d = bbox_transform_inv_3d(anchors_3d, bbox_deltas) # convert back to lidar_bv proposals_bv = lidar_3d_to_bv(proposals_3d) #[x1,y1,x2,y2] lidar_corners = lidar_3d_to_corners(proposals_3d) proposals_img = lidar_cnr_to_img(lidar_corners, calib[3], calib[2], calib[0]) if debug_state: # print "bbox_deltas: ", bbox_deltas[:10] # print "proposals number: ", proposals_3d[:10] print ("proposals_bv shape: ", proposals_bv.shape) print ("proposals_3d shape: ", proposals_3d.shape) print ("scores shape:", scores.shape) # 2. clip predicted boxes to image #WZN: delete those not in image ind_inside = clip_anchors(anchors, im_info[:2]) #ind_inside = np.logical_and(ind_inside,clip_anchors(proposals_bv, im_info[:2])) proposals_bv = proposals_bv[ind_inside,:] proposals_3d = proposals_3d[ind_inside,:] proposals_img = proposals_img[ind_inside,:] scores = scores[ind_inside,:] proposals_bv = clip_boxes(proposals_bv, im_info[:2]) # TODO: pass real image_info #keep = _filter_img_boxes(proposals_img, [375, 1242]) #proposals_bv = proposals_bv[keep, :] #proposals_3d = proposals_3d[keep, :] #proposals_img = proposals_img[keep, :] #scores = scores[keep] if debug_state: print ("proposals after clip") print ("proposals_bv shape: ", proposals_bv.shape) print ("proposals_3d shape: ", proposals_3d.shape) print ("proposals_img shape: ", proposals_img.shape) # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if cfg_in['pre_keep_topN'] > 0: order = order[:cfg_in['pre_keep_topN']] #keep = keep[order] proposals_bv = proposals_bv[order, :] proposals_3d = proposals_3d[order, :] proposals_img = proposals_img[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) if cfg_in['use_nms']: keep = nms(np.hstack((proposals_bv, scores)), cfg_in['nms_thresh']) if cfg_in['nms_topN'] > 0: keep = keep[:cfg_in['nms_topN']] proposals_bv = proposals_bv[keep, :] proposals_3d = proposals_3d[keep, :] proposals_img = proposals_img[keep, :] scores = scores[keep] if debug_state: print ("proposals after nms") print ("proposals_bv shape: ", proposals_bv.shape) print ("proposals_3d shape: ", proposals_3d.shape) # debug only: keep probabilities above a threshold if cfg_in['prob_thresh']: keep_ind = scores[:,0]>cfg_in['prob_thresh'] print ('scores: ',scores) print ('threshold: ', cfg_in['prob_thresh']) print ('score shape:', scores.shape) #print keep_ind.shape #print keep.shape #keep = keep[keep_ind] proposals_bv = proposals_bv[keep_ind, :] proposals_3d = proposals_3d[keep_ind, :] proposals_img = proposals_img[keep_ind, :] scores = scores[keep_ind] return proposals_bv,proposals_3d,proposals_img,scores
def _sample_rois_3d(all_rois_bv, all_rois_3d, gt_boxes_bv, gt_boxes_corners, fg_rois_per_image, rois_per_image, num_classes): """Generate a random sample of RoIs comprising foreground and background examples. """ # overlaps: (rois x gt_boxes) overlaps = bbox_overlaps( np.ascontiguousarray(all_rois_bv[:, 1:5], dtype=np.float), np.ascontiguousarray(gt_boxes_bv[:, :4], dtype=np.float)) gt_assignment = overlaps.argmax(axis=1) max_overlaps = overlaps.max(axis=1) labels = gt_boxes_bv[gt_assignment, 4] if DEBUG: print "overlaps: ", overlaps.shape print "gt assignment: ", gt_assignment.shape print "max_overlaps: ", max_overlaps.shape print "labels: ", labels.shape # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0] if DEBUG: print "fg_inds: ", fg_inds.shape # print "fg_rois_per_image: ", fg_rois_per_image # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = int(min(fg_rois_per_image, fg_inds.size)) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size) # Sample background regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Select sampled values from various arrays: labels = labels[keep_inds] # Clamp labels for the background RoIs to 0 labels[fg_rois_per_this_image:] = 0 rois_bv = all_rois_bv[keep_inds] rois_3d = all_rois_3d[keep_inds] # convert 3d to corners rois_cnr = lidar_3d_to_corners(rois_3d[:,1:7]) rois_cnr = np.hstack((rois_3d[:,0].reshape(-1,1), rois_cnr)) if DEBUG: print "labels shape: ", labels.shape print "keep_inds: ", keep_inds print "all_rois_bv shape:, ", all_rois_bv.shape print "rois_3d shape:, ", rois_3d.shape print "rois_cnr shape:, ", rois_cnr.shape bbox_target_data = _compute_targets_cnr( rois_cnr[:, 1:25], gt_boxes_corners[gt_assignment[keep_inds], :24], labels) bbox_targets = \ _get_bbox_regression_labels_3d(bbox_target_data, num_classes) return labels, rois_bv, rois_cnr, rois_3d, bbox_targets