def forward(self, input): # Algorithm: # # for each (H, W) location i # generate 9 anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the 9 anchors # filter out-of-image anchors rpn_cls_score = input[0] gt_boxes = input[1] im_info = input[2] num_boxes = input[3] # map of shape (..., H, W) height, width = rpn_cls_score.size(2), rpn_cls_score.size(3) batch_size = gt_boxes.size(0) feat_height, feat_width = rpn_cls_score.size(2), rpn_cls_score.size(3) shift_x = np.arange(0, feat_width) * self._feat_stride shift_y = np.arange(0, feat_height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = torch.from_numpy(np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose()) shifts = shifts.contiguous().type_as(rpn_cls_score).float() A = self._num_anchors K = shifts.size(0) self._anchors = self._anchors.type_as(gt_boxes) # move to specific gpu. all_anchors = self._anchors.view(1, A, 4) + shifts.view(K, 1, 4) all_anchors = all_anchors.view(K * A, 4) total_anchors = int(K * A) keep = ((all_anchors[:, 0] >= -self._allowed_border) & (all_anchors[:, 1] >= -self._allowed_border) & (all_anchors[:, 2] < long(im_info[0][1]) + self._allowed_border) & (all_anchors[:, 3] < long(im_info[0][0]) + self._allowed_border)) inds_inside = torch.nonzero(keep).view(-1) # keep only inside anchors anchors = all_anchors[inds_inside, :] # label: 1 is positive, 0 is negative, -1 is dont care labels = gt_boxes.new(batch_size, inds_inside.size(0)).fill_(-1) bbox_inside_weights = gt_boxes.new(batch_size, inds_inside.size(0)).zero_() bbox_outside_weights = gt_boxes.new(batch_size, inds_inside.size(0)).zero_() overlaps = bbox_overlaps_batch(anchors, gt_boxes) max_overlaps, argmax_overlaps = torch.max(overlaps, 2) gt_max_overlaps, _ = torch.max(overlaps, 1) if not cfg.TRAIN.RPN_CLOBBER_POSITIVES: labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 gt_max_overlaps[gt_max_overlaps==0] = 1e-5 keep = torch.sum(overlaps.eq(gt_max_overlaps.view(batch_size,1,-1).expand_as(overlaps)), 2) if torch.sum(keep) > 0: labels[keep>0] = 1 # fg label: above threshold IOU labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1 if cfg.TRAIN.RPN_CLOBBER_POSITIVES: labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE) sum_fg = torch.sum((labels == 1).int(), 1) sum_bg = torch.sum((labels == 0).int(), 1) for i in range(batch_size): # subsample positive labels if we have too many if sum_fg[i] > num_fg: fg_inds = torch.nonzero(labels[i] == 1).view(-1) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_inds.size(0)).type_as(gt_boxes).long() rand_num = torch.from_numpy(np.random.permutation(fg_inds.size(0))).type_as(gt_boxes).long() disable_inds = fg_inds[rand_num[:fg_inds.size(0)-num_fg]] labels[i][disable_inds] = -1 num_bg = cfg.TRAIN.RPN_BATCHSIZE - sum_fg[i] # subsample negative labels if we have too many if sum_bg[i] > num_bg: bg_inds = torch.nonzero(labels[i] == 0).view(-1) #rand_num = torch.randperm(bg_inds.size(0)).type_as(gt_boxes).long() rand_num = torch.from_numpy(np.random.permutation(bg_inds.size(0))).type_as(gt_boxes).long() disable_inds = bg_inds[rand_num[:bg_inds.size(0)-num_bg]] labels[i][disable_inds] = -1 offset = torch.arange(0, batch_size)*gt_boxes.size(1) argmax_overlaps = argmax_overlaps + offset.view(batch_size, 1).type_as(argmax_overlaps) bbox_targets = _compute_targets_batch(anchors, gt_boxes.view(-1,5)[argmax_overlaps.view(-1), :].view(batch_size, -1, 5)) # use a single value instead of 4 values for easy index. bbox_inside_weights[labels==1] = cfg.TRAIN.RPN_BBOX_INSIDE_WEIGHTS[0] if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0: num_examples = torch.sum(labels[i] >= 0) positive_weights = 1.0 / num_examples negative_weights = 1.0 / num_examples else: assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) & (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1)) bbox_outside_weights[labels == 1] = positive_weights bbox_outside_weights[labels == 0] = negative_weights labels = _unmap(labels, total_anchors, inds_inside, batch_size, fill=-1) bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, batch_size, fill=0) bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, batch_size, fill=0) bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, batch_size, fill=0) outputs = [] labels = labels.view(batch_size, height, width, A).permute(0,3,1,2).contiguous() labels = labels.view(batch_size, 1, A * height, width) outputs.append(labels) bbox_targets = bbox_targets.view(batch_size, height, width, A*4).permute(0,3,1,2).contiguous() outputs.append(bbox_targets) anchors_count = bbox_inside_weights.size(1) bbox_inside_weights = bbox_inside_weights.view(batch_size,anchors_count,1).expand(batch_size, anchors_count, 4) bbox_inside_weights = bbox_inside_weights.contiguous().view(batch_size, height, width, 4*A)\ .permute(0,3,1,2).contiguous() outputs.append(bbox_inside_weights) bbox_outside_weights = bbox_outside_weights.view(batch_size,anchors_count,1).expand(batch_size, anchors_count, 4) bbox_outside_weights = bbox_outside_weights.contiguous().view(batch_size, height, width, 4*A)\ .permute(0,3,1,2).contiguous() outputs.append(bbox_outside_weights) return outputs
def _sample_rois_pytorch(self, all_rois_left, all_rois_right, gt_boxes_left, gt_boxes_right, \ gt_dim_orien, gt_kpts, fg_rois_per_image, rois_per_image, num_classes): """Generate a random sample of RoIs comprising foreground and background examples. """ # overlaps: (rois x gt_boxes) overlaps_left = bbox_overlaps_batch( all_rois_left, gt_boxes_left) # B x num_rois x num_gt overlaps_right = bbox_overlaps_batch( all_rois_right, gt_boxes_right) # B x num_rois(2030) x num_gt(30) max_overlaps_left, gt_assignment_left = torch.max( overlaps_left, 2) # B x num_rois(2030) max_overlaps_right, gt_assignment_right = torch.max(overlaps_right, 2) batch_size = overlaps_left.size(0) num_proposal = overlaps_left.size(1) num_boxes_per_img = overlaps_left.size(2) offset = torch.arange(0, batch_size) * gt_boxes_left.size(1) offset = offset.view( -1, 1).type_as(gt_assignment_left) + gt_assignment_left labels = gt_boxes_left[:,:,4].contiguous().view(-1).index(offset.view(-1))\ .view(batch_size, -1) labels_batch = labels.new(batch_size, rois_per_image).zero_() rois_batch_left = all_rois_left.new(batch_size, rois_per_image, 5).zero_() gt_assign_batch_left = all_rois_left.new(batch_size, rois_per_image).zero_() gt_rois_batch_left = all_rois_left.new(batch_size, rois_per_image, 5).zero_() rois_batch_right = all_rois_right.new(batch_size, rois_per_image, 5).zero_() gt_assign_batch_right = all_rois_right.new(batch_size, rois_per_image).zero_() gt_rois_batch_right = all_rois_right.new(batch_size, rois_per_image, 5).zero_() gt_dim_orien_batch = all_rois_right.new(batch_size, rois_per_image, 5).zero_() gt_kpts_batch = all_rois_right.new(batch_size, rois_per_image, 6).zero_() # Guard against the case when an image has fewer than max_fg_rois_per_image # foreground RoIs for i in range(batch_size): fg_inds = torch.nonzero((max_overlaps_left[i] >= cfg.TRAIN.FG_THRESH) &\ (max_overlaps_right[i] >= cfg.TRAIN.FG_THRESH) & \ (gt_assignment_left[i] == gt_assignment_right[i])).view(-1) fg_num_rois = fg_inds.numel() # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds_left = torch.nonzero( (max_overlaps_left[i] < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps_left[i] >= cfg.TRAIN.BG_THRESH_LO)).view(-1) bg_inds_right = torch.nonzero( (max_overlaps_right[i] < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps_right[i] >= cfg.TRAIN.BG_THRESH_LO)).view(-1) bg_inds = torch.from_numpy( np.union1d(bg_inds_left.cpu().numpy(), bg_inds_right.cpu().numpy())).cuda() bg_num_rois = bg_inds.numel() if fg_num_rois > 0 and bg_num_rois > 0: # sampling fg fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_num_rois).long().cuda() rand_num = torch.from_numpy(np.random.permutation( fg_num_rois)).type_as(gt_boxes_left).long() fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]] # sampling bg bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image # Seems torch.rand has a bug, it will generate very large number and make an error. # We use numpy rand instead. #rand_num = (torch.rand(bg_rois_per_this_image) * bg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(bg_rois_per_this_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as( gt_boxes_left).long() bg_inds = bg_inds[rand_num] elif fg_num_rois > 0 and bg_num_rois == 0: # sampling fg #rand_num = torch.floor(torch.rand(rois_per_image) * fg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(rois_per_image) * fg_num_rois) rand_num = torch.from_numpy(rand_num).type_as( gt_boxes_left).long() fg_inds = fg_inds[rand_num] fg_rois_per_this_image = rois_per_image bg_rois_per_this_image = 0 elif bg_num_rois > 0 and fg_num_rois == 0: # sampling bg #rand_num = torch.floor(torch.rand(rois_per_image) * bg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(rois_per_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as( gt_boxes_left).long() bg_inds = bg_inds[rand_num] bg_rois_per_this_image = rois_per_image fg_rois_per_this_image = 0 else: raise ValueError( "bg_num_rois = 0 and fg_num_rois = 0, this should not happen!" ) # The indices that we're selecting (both fg and bg) keep_inds = torch.cat([fg_inds, bg_inds], 0) # Select sampled values from various arrays: labels_batch[i].copy_(labels[i][keep_inds]) # Clamp labels for the background RoIs to 0 labels_batch[i][fg_rois_per_this_image:] = 0 rois_batch_left[i] = all_rois_left[i][keep_inds] rois_batch_left[i, :, 0] = i rois_batch_right[i] = all_rois_right[i][keep_inds] rois_batch_right[i, :, 0] = i # TODO: check the below line when batch_size > 1, no need to add offset here gt_assign_batch_left[i] = gt_assignment_left[i][keep_inds] gt_assign_batch_right[i] = gt_assignment_right[i][keep_inds] gt_rois_batch_left[i] = gt_boxes_left[i][gt_assignment_left[i] [keep_inds]] gt_rois_batch_right[i] = gt_boxes_right[i][gt_assignment_right[i] [keep_inds]] gt_dim_orien_batch[i] = gt_dim_orien[i][gt_assignment_left[i] [keep_inds]] gt_kpts_batch[i] = gt_kpts[i][gt_assignment_left[i][keep_inds]] bbox_target_data_left = self._compute_targets_pytorch( rois_batch_left[:, :, 1:5], gt_rois_batch_left[:, :, :4]) bbox_target_data_right = self._compute_targets_pytorch( rois_batch_right[:, :, 1:5], gt_rois_batch_right[:, :, :4]) dim_orien_target_data = self._compute_dim_orien_targets_pytorch( gt_dim_orien_batch) kpts_target_data, kpts_weight = self._compute_kpts_targets_pytorch( rois_batch_left[:, :, 1:5], gt_kpts_batch) bbox_targets_left, bbox_inside_weights_left = \ self._get_bbox_regression_labels_pytorch(bbox_target_data_left, labels_batch, num_classes) bbox_targets_right, bbox_inside_weights_right = \ self._get_bbox_regression_labels_pytorch(bbox_target_data_right, labels_batch, num_classes) dim_orien_target = self._get_dim_orien_regression_labels_pytorch( dim_orien_target_data, labels_batch, num_classes) kpts_targets, kpts_weight = self._get_kpts_regression_labels_pytorch( kpts_target_data, labels_batch, num_classes, kpts_weight) return labels_batch, rois_batch_left, rois_batch_right, gt_assign_batch_left, \ bbox_targets_left, bbox_targets_right, dim_orien_target, kpts_targets, kpts_weight, bbox_inside_weights_left
def forward(self, all_rois, gt_boxes, num_boxes): self.BBOX_NORMALIZE_MEANS = self.BBOX_NORMALIZE_MEANS.type_as(gt_boxes) self.BBOX_NORMALIZE_STDS = self.BBOX_NORMALIZE_STDS.type_as(gt_boxes) self.BBOX_INSIDE_WEIGHTS = self.BBOX_INSIDE_WEIGHTS.type_as(gt_boxes) gt_boxes_append = gt_boxes.new(gt_boxes.size()).zero_() gt_boxes_append[:, :, 1:5] = gt_boxes[:, :, :4] # Include ground-truth boxes in the set of candidate rois all_rois = torch.cat([all_rois, gt_boxes_append], 1) num_images = 1 rois_per_image = int(self.BATCH_SIZE / num_images) fg_rois_per_image = int(np.round(self.FG_FRACTION * rois_per_image)) fg_rois_per_image = 1 if fg_rois_per_image == 0 else fg_rois_per_image # sample function was here overlaps = bbox_overlaps_batch(all_rois, gt_boxes) max_overlaps, gt_assignment = torch.max(overlaps, 2) batch_size = overlaps.size(0) num_proposal = overlaps.size(1) num_boxes_per_img = overlaps.size(2) offset = torch.arange(0, batch_size) * gt_boxes.size(1) offset = offset.view(-1, 1).type_as(gt_assignment) + gt_assignment # changed indexing way for pytorch 1.0 labels = gt_boxes[:, :, 4].contiguous().view(-1)[(offset.view(-1), )].view( batch_size, -1) labels_batch = labels.new(batch_size, rois_per_image).zero_() rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_() gt_rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_() # Guard against the case when an image has fewer than max_fg_rois_per_image # foreground RoIs for i in range(batch_size): fg_inds = torch.nonzero(max_overlaps[i] >= self.FG_THRESH).view(-1) fg_num_rois = fg_inds.numel() # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = torch.nonzero((max_overlaps[i] < self.BG_THRESH_HI) & ( max_overlaps[i] >= self.BG_THRESH_LO)).view(-1) bg_num_rois = bg_inds.numel() if fg_num_rois > 0 and bg_num_rois > 0: # sampling fg fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_num_rois).long().cuda() rand_num = torch.from_numpy(np.random.permutation( fg_num_rois)).type_as(gt_boxes).long() fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]] # sampling bg bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image # Seems torch.rand has a bug, it will generate very large number and make an error. # We use numpy rand instead. #rand_num = (torch.rand(bg_rois_per_this_image) * bg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(bg_rois_per_this_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() bg_inds = bg_inds[rand_num] elif fg_num_rois > 0 and bg_num_rois == 0: # sampling fg #rand_num = torch.floor(torch.rand(rois_per_image) * fg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(rois_per_image) * fg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() fg_inds = fg_inds[rand_num] fg_rois_per_this_image = rois_per_image bg_rois_per_this_image = 0 elif bg_num_rois > 0 and fg_num_rois == 0: # sampling bg #rand_num = torch.floor(torch.rand(rois_per_image) * bg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(rois_per_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() bg_inds = bg_inds[rand_num] bg_rois_per_this_image = rois_per_image fg_rois_per_this_image = 0 else: raise ValueError( "bg_num_rois = 0 and fg_num_rois = 0, this should not happen!" ) # The indices that we're selecting (both fg and bg) keep_inds = torch.cat([fg_inds, bg_inds], 0) # Select sampled values from various arrays: labels_batch[i].copy_(labels[i][keep_inds]) # Clamp labels for the background RoIs to 0 if fg_rois_per_this_image < rois_per_image: labels_batch[i][fg_rois_per_this_image:] = 0 rois_batch[i] = all_rois[i][keep_inds] rois_batch[i, :, 0] = i gt_rois_batch[i] = gt_boxes[i][gt_assignment[i][keep_inds]] matches = labels_batch rois = rois_batch gt_rois = gt_rois_batch return gt_rois, rois, matches
def forward(self, input): # Algorithm: # # for each (H, W) location i # generate 9 anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the 9 anchors # filter out-of-image anchors rpn_cls_score = input[0] gt_boxes = input[1] im_info = input[2] num_boxes = input[3] # map of shape (..., H, W) height, width = rpn_cls_score.size(2), rpn_cls_score.size(3) batch_size = gt_boxes.size(0) feat_height, feat_width = rpn_cls_score.size(2), rpn_cls_score.size(3) shift_x = np.arange(0, feat_width) * self._feat_stride shift_y = np.arange(0, feat_height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = torch.from_numpy( np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose()) shifts = shifts.contiguous().type_as(rpn_cls_score).float() A = self._num_anchors K = shifts.size(0) self._anchors = self._anchors.type_as( gt_boxes) # move to specific gpu. all_anchors = self._anchors.view(1, A, 4) + shifts.view(K, 1, 4) all_anchors = all_anchors.view(K * A, 4) total_anchors = int(K * A) keep = ( (all_anchors[:, 0] >= -self._allowed_border) & (all_anchors[:, 1] >= -self._allowed_border) & (all_anchors[:, 2] < long(im_info[0][1]) + self._allowed_border) & (all_anchors[:, 3] < long(im_info[0][0]) + self._allowed_border)) inds_inside = torch.nonzero(keep).view(-1) # keep only inside anchors anchors = all_anchors[inds_inside, :] # label: 1 is positive, 0 is negative, -1 is dont care labels = gt_boxes.new(batch_size, inds_inside.size(0)).fill_(-1) bbox_inside_weights = gt_boxes.new(batch_size, inds_inside.size(0)).zero_() bbox_outside_weights = gt_boxes.new(batch_size, inds_inside.size(0)).zero_() overlaps = bbox_overlaps_batch(anchors, gt_boxes) max_overlaps, argmax_overlaps = torch.max(overlaps, 2) gt_max_overlaps, _ = torch.max(overlaps, 1) if not cfg.TRAIN.RPN_CLOBBER_POSITIVES: labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 gt_max_overlaps[gt_max_overlaps == 0] = 1e-5 keep = torch.sum( overlaps.eq( gt_max_overlaps.view(batch_size, 1, -1).expand_as(overlaps)), 2) if torch.sum(keep) > 0: labels[keep > 0] = 1 # fg label: above threshold IOU labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1 if cfg.TRAIN.RPN_CLOBBER_POSITIVES: labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE) sum_fg = torch.sum((labels == 1).int(), 1) sum_bg = torch.sum((labels == 0).int(), 1) for i in range(batch_size): # subsample positive labels if we have too many if sum_fg[i] > num_fg: fg_inds = torch.nonzero(labels[i] == 1).view(-1) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_inds.size(0)).type_as(gt_boxes).long() rand_num = torch.from_numpy( np.random.permutation( fg_inds.size(0))).type_as(gt_boxes).long() disable_inds = fg_inds[rand_num[:fg_inds.size(0) - num_fg]] labels[i][disable_inds] = -1 num_bg = cfg.TRAIN.RPN_BATCHSIZE - sum_fg[i] # subsample negative labels if we have too many if sum_bg[i] > num_bg: bg_inds = torch.nonzero(labels[i] == 0).view(-1) #rand_num = torch.randperm(bg_inds.size(0)).type_as(gt_boxes).long() rand_num = torch.from_numpy( np.random.permutation( bg_inds.size(0))).type_as(gt_boxes).long() disable_inds = bg_inds[rand_num[:bg_inds.size(0) - num_bg]] labels[i][disable_inds] = -1 offset = torch.arange(0, batch_size) * gt_boxes.size(1) argmax_overlaps = argmax_overlaps + offset.view( batch_size, 1).type_as(argmax_overlaps) bbox_targets = _compute_targets_batch( anchors, gt_boxes.view(-1, 5)[argmax_overlaps.view(-1), :].view( batch_size, -1, 5)) # use a single value instead of 4 values for easy index. bbox_inside_weights[labels == 1] = cfg.TRAIN.RPN_BBOX_INSIDE_WEIGHTS[0] if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0: num_examples = torch.sum(labels[i] >= 0) positive_weights = 1.0 / num_examples negative_weights = 1.0 / num_examples else: assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) & (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1)) bbox_outside_weights[labels == 1] = positive_weights bbox_outside_weights[labels == 0] = negative_weights labels = _unmap(labels, total_anchors, inds_inside, batch_size, fill=-1) bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, batch_size, fill=0) bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, batch_size, fill=0) bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, batch_size, fill=0) outputs = [] labels = labels.view(batch_size, height, width, A).permute(0, 3, 1, 2).contiguous() labels = labels.view(batch_size, 1, A * height, width) outputs.append(labels) bbox_targets = bbox_targets.view(batch_size, height, width, A * 4).permute(0, 3, 1, 2).contiguous() outputs.append(bbox_targets) anchors_count = bbox_inside_weights.size(1) bbox_inside_weights = bbox_inside_weights.view( batch_size, anchors_count, 1).expand(batch_size, anchors_count, 4) bbox_inside_weights = bbox_inside_weights.contiguous().view(batch_size, height, width, 4*A)\ .permute(0,3,1,2).contiguous() outputs.append(bbox_inside_weights) bbox_outside_weights = bbox_outside_weights.view( batch_size, anchors_count, 1).expand(batch_size, anchors_count, 4) bbox_outside_weights = bbox_outside_weights.contiguous().view(batch_size, height, width, 4*A)\ .permute(0,3,1,2).contiguous() outputs.append(bbox_outside_weights) return outputs
def _sample_rois_pytorch(self, all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes): """Generate a random sample of RoIs comprising foreground and background examples. """ # overlaps: (rois x gt_boxes) overlaps = bbox_overlaps_batch(all_rois, gt_boxes) max_overlaps, gt_assignment = torch.max(overlaps, 2) batch_size = overlaps.size(0) num_proposal = overlaps.size(1) num_boxes_per_img = overlaps.size(2) offset = torch.arange(0, batch_size)*gt_boxes.size(1) offset = offset.view(-1, 1).type_as(gt_assignment) + gt_assignment labels = gt_boxes[:,:,4].contiguous().view(-1).index(offset.view(-1))\ .view(batch_size, -1) labels_batch = labels.new(batch_size, rois_per_image).zero_() rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_() gt_rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_() # Guard against the case when an image has fewer than max_fg_rois_per_image # foreground RoIs for i in range(batch_size): fg_inds = torch.nonzero(max_overlaps[i] >= cfg.TRAIN.FG_THRESH).view(-1) fg_num_rois = fg_inds.numel() # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = torch.nonzero((max_overlaps[i] < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps[i] >= cfg.TRAIN.BG_THRESH_LO)).view(-1) bg_num_rois = bg_inds.numel() if fg_num_rois > 0 and bg_num_rois > 0: # sampling fg fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_num_rois).long().cuda() rand_num = torch.from_numpy(np.random.permutation(fg_num_rois)).type_as(gt_boxes).long() fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]] # sampling bg bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image # Seems torch.rand has a bug, it will generate very large number and make an error. # We use numpy rand instead. #rand_num = (torch.rand(bg_rois_per_this_image) * bg_num_rois).long().cuda() rand_num = np.floor(np.random.rand(bg_rois_per_this_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() bg_inds = bg_inds[rand_num] elif fg_num_rois > 0 and bg_num_rois == 0: # sampling fg #rand_num = torch.floor(torch.rand(rois_per_image) * fg_num_rois).long().cuda() rand_num = np.floor(np.random.rand(rois_per_image) * fg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() fg_inds = fg_inds[rand_num] fg_rois_per_this_image = rois_per_image bg_rois_per_this_image = 0 elif bg_num_rois > 0 and fg_num_rois == 0: # sampling bg #rand_num = torch.floor(torch.rand(rois_per_image) * bg_num_rois).long().cuda() rand_num = np.floor(np.random.rand(rois_per_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() bg_inds = bg_inds[rand_num] bg_rois_per_this_image = rois_per_image fg_rois_per_this_image = 0 else: raise ValueError("bg_num_rois = 0 and fg_num_rois = 0, this should not happen!") # The indices that we're selecting (both fg and bg) keep_inds = torch.cat([fg_inds, bg_inds], 0) # Select sampled values from various arrays: labels_batch[i].copy_(labels[i][keep_inds]) # Clamp labels for the background RoIs to 0 if fg_rois_per_this_image < rois_per_image: labels_batch[i][fg_rois_per_this_image:] = 0 rois_batch[i] = all_rois[i][keep_inds] rois_batch[i,:,0] = i gt_rois_batch[i] = gt_boxes[i][gt_assignment[i][keep_inds]] bbox_target_data = self._compute_targets_pytorch( rois_batch[:,:,1:5], gt_rois_batch[:,:,:4]) bbox_targets, bbox_inside_weights = \ self._get_bbox_regression_labels_pytorch(bbox_target_data, labels_batch, num_classes) return labels_batch, rois_batch, bbox_targets, bbox_inside_weights
def _sample_rois_pytorch(self, all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes): """Generate a random sample of RoIs comprising foreground and background examples. """ # overlaps: (rois x gt_boxes) overlaps = bbox_overlaps_batch(all_rois, gt_boxes) max_overlaps, gt_assignment = torch.max(overlaps, 2) batch_size = overlaps.size(0) num_proposal = overlaps.size(1) num_boxes_per_img = overlaps.size(2) offset = torch.arange(0, batch_size) * gt_boxes.size(1) offset = offset.view(-1, 1).type_as(gt_assignment) + gt_assignment labels = gt_boxes[:,:,4].contiguous().view(-1).index(offset.view(-1))\ .view(batch_size, -1) labels_batch = labels.new(batch_size, rois_per_image).zero_() ## size of: 4x128 # print("inside0, labels_batch :", labels_batch, "labels", labels, "gt_box:", gt_boxes) rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_() gt_rois_batch = all_rois.new(batch_size, rois_per_image, 5).zero_() # Guard against the case when an image has fewer than max_fg_rois_per_image # foreground RoIs for i in range(batch_size): fg_inds = torch.nonzero( max_overlaps[i] >= cfg.TRAIN.FG_THRESH).view(-1) fg_num_rois = fg_inds.numel() # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds = torch.nonzero( (max_overlaps[i] < cfg.TRAIN.BG_THRESH_HI) & (max_overlaps[i] >= cfg.TRAIN.BG_THRESH_LO)).view(-1) bg_num_rois = bg_inds.numel() # print("inside01, bg_num_rois :", bg_num_rois, fg_num_rois, fg_rois_per_image) if fg_num_rois > 0 and bg_num_rois > 0: # sampling fg fg_rois_per_this_image = min(fg_rois_per_image, fg_num_rois) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_num_rois).long().cuda() rand_num = torch.from_numpy(np.random.permutation( fg_num_rois)).type_as(gt_boxes).long() # print("inside0122:",rand_num) fg_inds = fg_inds[rand_num[:fg_rois_per_this_image]] # print("inside0123:",fg_inds, rois_per_image, fg_rois_per_this_image) # sampling bg bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image # print("inside0123:",bg_rois_per_this_image) # Seems torch.rand has a bug, it will generate very large number and make an error. # We use numpy rand instead. #rand_num = (torch.rand(bg_rois_per_this_image) * bg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(bg_rois_per_this_image) * bg_num_rois) # print("inside0124:",rand_num) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() # print("inside0125:",rand_num) bg_inds = bg_inds[rand_num] # print("inside0126:",bg_inds) elif fg_num_rois > 0 and bg_num_rois == 0: # sampling fg #rand_num = torch.floor(torch.rand(rois_per_image) * fg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(rois_per_image) * fg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() fg_inds = fg_inds[rand_num] fg_rois_per_this_image = rois_per_image bg_rois_per_this_image = 0 elif bg_num_rois > 0 and fg_num_rois == 0: # sampling bg #rand_num = torch.floor(torch.rand(rois_per_image) * bg_num_rois).long().cuda() rand_num = np.floor( np.random.rand(rois_per_image) * bg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes).long() bg_inds = bg_inds[rand_num] bg_rois_per_this_image = rois_per_image fg_rois_per_this_image = 0 else: raise ValueError( "bg_num_rois = 0 and fg_num_rois = 0, this should not happen!" ) # print("inside02, bg_num_rois :", fg_inds, " bg_inds ", bg_inds) # The indices that we're selecting (both fg and bg) keep_inds = torch.cat([fg_inds, bg_inds], 0) # Select sampled values from various arrays: labels_batch[i].copy_(labels[i][keep_inds]) # print("inside1, labels_batch :", labels_batch) # Clamp labels for the background RoIs to 0 if fg_rois_per_this_image < rois_per_image: labels_batch[i][fg_rois_per_this_image:] = 0 # print("inside2, labels_batch :", labels_batch) rois_batch[i] = all_rois[i][keep_inds] rois_batch[i, :, 0] = i gt_rois_batch[i] = gt_boxes[i][gt_assignment[i][keep_inds]] bbox_target_data = self._compute_targets_pytorch( rois_batch[:, :, 1:5], gt_rois_batch[:, :, :4]) bbox_targets, bbox_inside_weights = \ self._get_bbox_regression_labels_pytorch(bbox_target_data, labels_batch, num_classes) # print("inside3, labels_batch :", labels_batch) return labels_batch, rois_batch, bbox_targets, bbox_inside_weights
def forward(self, input): # Algorithm: # # for each (H, W) location i # generate 9 anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the 9 anchors # filter out-of-image anchors scores = input[0] gt_boxes_left = input[1] gt_boxes_right = input[2] gt_boxes_merge = input[3] im_info = input[4] num_boxes = input[5] feat_shapes = input[6] # NOTE: need to change # height, width = scores.size(2), scores.size(3) height, width = 0, 0 batch_size = gt_boxes_left.size(0) anchors = torch.from_numpy( generate_anchors_all_pyramids( self._fpn_scales, self._anchor_ratios, feat_shapes, self._fpn_feature_strides, self._fpn_anchor_stride)).type_as(scores) total_anchors = anchors.size(0) keep = ((anchors[:, 0] >= -self._allowed_border) & (anchors[:, 1] >= -self._allowed_border) & (anchors[:, 2] < long(im_info[0][1]) + self._allowed_border) & (anchors[:, 3] < long(im_info[0][0]) + self._allowed_border)) inds_inside = torch.nonzero(keep).view(-1) # keep only inside anchors anchors = anchors[inds_inside, :] # label: 1 is positive, 0 is negative, -1 is dont care merged_labels = gt_boxes_left.new(batch_size, inds_inside.size(0)).fill_(-1) bbox_inside_weights = gt_boxes_left.new(batch_size, inds_inside.size(0)).zero_() bbox_outside_weights = gt_boxes_left.new(batch_size, inds_inside.size(0)).zero_() overlaps = bbox_overlaps_batch(anchors, gt_boxes_merge) max_overlaps, argmax_overlaps = torch.max(overlaps, 2) gt_max_overlaps, _ = torch.max(overlaps, 1) if not cfg.TRAIN.RPN_CLOBBER_POSITIVES: merged_labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 gt_max_overlaps[gt_max_overlaps == 0] = 1e-5 keep = torch.sum( overlaps.eq( gt_max_overlaps.view(batch_size, 1, -1).expand_as(overlaps)), 2) if torch.sum(keep) > 0: merged_labels[keep > 0] = 1 # fg label: above threshold IOU merged_labels[max_overlaps >= cfg.TRAIN.RPN_POSITIVE_OVERLAP] = 1 if cfg.TRAIN.RPN_CLOBBER_POSITIVES: merged_labels[max_overlaps < cfg.TRAIN.RPN_NEGATIVE_OVERLAP] = 0 num_fg = int(cfg.TRAIN.RPN_FG_FRACTION * cfg.TRAIN.RPN_BATCHSIZE) sum_fg = torch.sum((merged_labels == 1).int(), 1) sum_bg = torch.sum((merged_labels == 0).int(), 1) for i in range(batch_size): # subsample positive merged_labels if we have too many if sum_fg[i] > num_fg: fg_inds = torch.nonzero(merged_labels[i] == 1).view(-1) # torch.randperm seems has a bug on multi-gpu setting that cause the segfault. # See https://github.com/pytorch/pytorch/issues/1868 for more details. # use numpy instead. #rand_num = torch.randperm(fg_inds.size(0)).type_as(gt_boxes_left).long() rand_num = torch.from_numpy( np.random.permutation( fg_inds.size(0))).type_as(gt_boxes_left).long() disable_inds = fg_inds[rand_num[:fg_inds.size(0) - num_fg]] merged_labels[i][disable_inds] = -1 num_bg = cfg.TRAIN.RPN_BATCHSIZE - sum_fg[i] # subsample negative merged_labels if we have too many if sum_bg[i] > num_bg: bg_inds = torch.nonzero(merged_labels[i] == 0).view(-1) #rand_num = torch.randperm(bg_inds.size(0)).type_as(gt_boxes_left).long() rand_num = torch.from_numpy( np.random.permutation( bg_inds.size(0))).type_as(gt_boxes_left).long() disable_inds = bg_inds[rand_num[:bg_inds.size(0) - num_bg]] merged_labels[i][disable_inds] = -1 offset = torch.arange(0, batch_size) * gt_boxes_left.size(1) argmax_overlaps = argmax_overlaps + offset.view( batch_size, 1).type_as(argmax_overlaps) bbox_targets_left = _compute_targets_batch( anchors, gt_boxes_left.view(-1, 5)[argmax_overlaps.view(-1), :].view( batch_size, -1, 5)) bbox_targets_right = _compute_targets_batch( anchors, gt_boxes_right.view(-1, 5)[argmax_overlaps.view(-1), :].view( batch_size, -1, 5)) # use a single value instead of 4 values for easy index. bbox_inside_weights[merged_labels == 1] = cfg.TRAIN.RPN_BBOX_INSIDE_WEIGHTS[0] if cfg.TRAIN.RPN_POSITIVE_WEIGHT < 0: num_examples = torch.sum(merged_labels[i] >= 0) positive_weights = 1.0 / num_examples negative_weights = 1.0 / num_examples else: assert ((cfg.TRAIN.RPN_POSITIVE_WEIGHT > 0) & (cfg.TRAIN.RPN_POSITIVE_WEIGHT < 1)) bbox_outside_weights[merged_labels == 1] = positive_weights bbox_outside_weights[merged_labels == 0] = negative_weights merged_labels = _unmap(merged_labels, total_anchors, inds_inside, batch_size, fill=-1) bbox_targets_left = _unmap(bbox_targets_left, total_anchors, inds_inside, batch_size, fill=0) bbox_targets_right = _unmap(bbox_targets_right, total_anchors, inds_inside, batch_size, fill=0) bbox_inside_weights = _unmap(bbox_inside_weights, total_anchors, inds_inside, batch_size, fill=0) bbox_outside_weights = _unmap(bbox_outside_weights, total_anchors, inds_inside, batch_size, fill=0) outputs = [] outputs.append(merged_labels) outputs.append(bbox_targets_left) outputs.append(bbox_targets_right) outputs.append(bbox_inside_weights) outputs.append(bbox_outside_weights) return outputs
float(data['objects'][j]['bbox'][3])) > 0.65 and float( data['objects'][j]['bbox'][3] ) > 50 and data['objects'][j]['label'] == 'pedestrian': box = [ float(data['objects'][j]['bbox'][0]), float(data['objects'][j]['bbox'][1]), float(data['objects'][j]['bbox'][0]) + float(data['objects'][j]['bbox'][2]) - 1, float(data['objects'][j]['bbox'][1]) + float(data['objects'][j]['bbox'][3]) - 1, 1.0 ] boxes.append(box) if len(boxes) > 0: boxes_tensor = torch.zeros(1, len(boxes), 5) boxes_tensor[0, :, :] = torch.from_numpy(np.array(boxes)).float() box_anchor_iou = bbox_overlaps_batch( all_anchors, boxes_tensor.view(1, len(boxes), 5)) gt_max_overlaps, _ = torch.max(box_anchor_iou, 1) print(gt_max_overlaps) total_iou += gt_max_overlaps.sum() num_fg += gt_max_overlaps.shape[1] gt_over07 += int(torch.sum(gt_max_overlaps > 0.7)) total_over07 += int(torch.sum(box_anchor_iou > 0.7)) total_iou07 += float(torch.sum(box_anchor_iou[box_anchor_iou > 0.7])) print('number of positive labels: ' + str(num_fg)) print('average IOU of max IOU per gt: ' + str(total_iou / num_fg)) print('recall: ' + str(float(gt_over07) / num_fg)) print('average number of positive anchors: ' + str(float(total_over07) / num_fg)) if total_over07 > 0: print('average IOU over 0.7: ' + str(total_iou07 / total_over07))