def anchor_iou_target_opr(self, boxes, im_info, all_anchors, rpn_bbox_offsets): n = rpn_bbox_offsets.shape[0] res = [] for i in range(n): gtboxes = boxes[i, :im_info[i, 5].astype(np.int32)] offsets = rpn_bbox_offsets[i].reshape(-1, 4).detach() m = offsets.shape[0] an, ac = all_anchors.shape[0], all_anchors.shape[1] anchors = F.broadcast_to(F.expand_dims(all_anchors, 1), (an, 1, ac)).reshape(-1, ac) dtboxes = bbox_transform_inv_opr(anchors[:, :4], offsets[:, :4]) overlaps = box_overlap_opr(dtboxes, gtboxes[:, :4]) ignore_mask = 1 - F.equal( gtboxes[:, 4], config.anchor_ignore_label).astype(np.float32) ignore_mask = F.expand_dims(ignore_mask, axis=0) overlaps = overlaps * ignore_mask index = F.argmax(overlaps, axis=1) value = F.nn.indexing_one_hot(overlaps, index, 1) value = F.expand_dims(F.expand_dims(value, axis=1), axis=0) res.append(value) result = F.concat(res, 0) return result
def fpn_anchor_target_opr_core_impl(gt_boxes, im_info, anchors, allow_low_quality_matches=True): ignore_label = config.ignore_label # get the gt boxes gtboxes = gt_boxes[:im_info[5].astype(np.int32)] ignore_mask = F.equal(gtboxes[:, 4], config.ignore_label) # find the valid gtboxes _, index = F.cond_take(1 - ignore_mask > 0, ignore_mask) valid_gt_boxes = gtboxes[index.astype(np.int32)] # compute the iou matrix overlaps = box_overlap_opr(anchors, valid_gt_boxes[:, :4]) # match the dtboxes a_shp0 = anchors.shape[0] argmax_overlaps = F.argmax(overlaps, axis=1) max_overlaps = F.nn.indexing_one_hot(overlaps, argmax_overlaps.astype(np.int32), 1) labels = F.ones(a_shp0).astype(np.int32) * ignore_label # set negative ones labels = labels * (max_overlaps >= config.rpn_negative_overlap).astype( np.float32) # set positive ones fg_mask = (max_overlaps >= config.rpn_positive_overlap) const_one = mge.tensor(1.0) if allow_low_quality_matches: # match the max gt gt_max_overlaps = F.max(overlaps, axis=0) gt_argmax_overlaps = F.argmax(overlaps, axis=0) gt_argmax_overlaps = gt_argmax_overlaps.astype(np.int32) max_overlaps[gt_argmax_overlaps] = 1. m = gt_max_overlaps.shape[0] argmax_overlaps[gt_argmax_overlaps] = F.linspace(0, m - 1, m).astype(np.int32) fg_mask = (max_overlaps >= config.rpn_positive_overlap) labels[fg_mask] = 1 # compute the bbox targets bbox_targets = bbox_transform_opr(anchors, valid_gt_boxes[argmax_overlaps, :4]) if config.rpn_bbox_normalize_targets: std_opr = mge.tensor(config.bbox_normalize_stds[None, :]).to( anchors.device) mean_opr = mge.tensor(config.bbox_normalize_means[None, :]).to( anchors.device) minus_opr = mean_opr / std_opr bbox_targets = bbox_targets / std_opr - minus_opr return labels, bbox_targets
def fpn_anchor_target_opr_core_impl(gt_boxes, im_info, anchors, allow_low_quality_matches=True): ignore_label = config.ignore_label # get the gt boxes valid_gt_boxes = gt_boxes[:im_info[5], :] non_ignore_mask = valid_gt_boxes[:, -1] > 0 non_ignore_inds = mask_to_inds(non_ignore_mask) valid_gt_boxes = valid_gt_boxes.ai[non_ignore_inds] # compute the iou matrix overlaps = box_overlap_opr(anchors, valid_gt_boxes[:, :4]) # match the dtboxes a_shp0 = anchors.shape[0] max_overlaps = F.max(overlaps, axis=1) argmax_overlaps = F.argmax(overlaps, axis=1) # all ignore labels = mge.ones(a_shp0).astype(np.int32) * ignore_label # set negative ones labels = labels * (max_overlaps >= config.rpn_negative_overlap) # set positive ones fg_mask = (max_overlaps >= config.rpn_positive_overlap) const_one = mge.tensor(1.0) if allow_low_quality_matches: # match the max gt gt_max_overlaps = F.max(overlaps, axis=0) gt_argmax_overlaps = F.argmax(overlaps, axis=0) g_shp0 = valid_gt_boxes.shapeof()[0] gt_id = F.linspace(0, g_shp0 - 1, g_shp0).astype(np.int32) argmax_overlaps = argmax_overlaps.set_ai(gt_id)[gt_argmax_overlaps] max_overlaps = max_overlaps.set_ai( const_one.broadcast(g_shp0))[gt_argmax_overlaps] fg_mask = (max_overlaps >= config.rpn_positive_overlap) # set positive ones fg_mask_ind = mask_to_inds(fg_mask) labels = labels.set_ai(const_one.broadcast( fg_mask_ind.shapeof()))[fg_mask_ind] # compute the targets bbox_targets = bbox_transform_opr(anchors, valid_gt_boxes.ai[argmax_overlaps, :4]) if config.rpn_bbox_normalize_targets: std_opr = mge.tensor(config.bbox_normalize_stds[None, :]) mean_opr = mge.tensor(config.bbox_normalize_means[None, :]) minus_opr = mean_opr / std_opr bbox_targets = bbox_targets / std_opr - minus_opr return labels, bbox_targets
def _anchor_double_target(gt_boxes, im_info, all_anchors): gt_boxes, im_info = gt_boxes.detach(), im_info.detach() all_anchors = all_anchors.detach() gt_boxes = gt_boxes[:im_info[5].astype(np.int32), :] dummy = -F.ones([1, gt_boxes.shape[1]]).to(gt_boxes.device) gt_boxes = F.concat([gt_boxes, dummy], axis=0) valid_mask = 1 - (gt_boxes[:, 4] < 0).astype(np.float32) anchor_centers = _compute_center(all_anchors) gtboxes_centers = _compute_center(gt_boxes) # gtboxes_centers = gtboxes_centers * valid_mask.unsqueeze(1) gtboxes_centers = gtboxes_centers * F.expand_dims(valid_mask, axis=1) N, K = all_anchors.shape[0], gt_boxes.shape[0] an_centers = F.expand_dims(anchor_centers, axis=1) gt_centers = F.expand_dims(gtboxes_centers, axis=0) # an_centers = anchor_centers.unsqueeze(1).repeat(1, K, 1) # gt_centers = gtboxes_centers.unsqueeze(0).repeat(N, 1, 1) distance = F.abs(an_centers - gt_centers) distance = F.sqrt(F.pow(distance, 2).sum(axis=2)) start = 0 end = 5 overlaps = box_overlap_opr(all_anchors[:, :4], gt_boxes[:, :4]) overlaps *= F.expand_dims(valid_mask, axis=0) default_num = 16 ious_list = [] for l in range(start, end): _, index = F.cond_take(all_anchors[:, 4] == l, all_anchors[:, 4]) level_dist = distance[index, :].transpose(1, 0) ious = overlaps[index, :].transpose(1, 0) sorted_index = F.argsort(level_dist, descending=False) n = min(sorted_index.shape[1], default_num) ious = F.gather(ious, 1, sorted_index[:, :n]).transpose(1, 0) ious_list.append(ious) ious = F.concat(ious_list, axis=0) mean_var = F.mean(ious, axis=0) std_var = F.std(ious, 0) iou_thresh_per_gt = mean_var + std_var iou_thresh_per_gt = F.maximum(iou_thresh_per_gt, 0.2) # limits the anchor centers in the gtboxes N, K = all_anchors.shape[0], gt_boxes.shape[0] anchor_points = an_centers pos_area = _compute_pos_area(gt_boxes, 0.3) # pos_area = pos_area.unsqueeze(0).repeat(N, 1, 1) pos_area = F.broadcast_to(F.expand_dims(pos_area, axis=0), (N, K, pos_area.shape[-1])) l = anchor_points[:, :, 0] - pos_area[:, :, 0] r = pos_area[:, :, 2] - anchor_points[:, :, 0] t = anchor_points[:, :, 1] - pos_area[:, :, 1] b = pos_area[:, :, 3] - anchor_points[:, :, 1] is_in_gt = F.stack([l, r, t, b], axis=2) is_in_gt = is_in_gt.min(axis=2) > 0.1 valid_mask = (overlaps >= F.expand_dims( iou_thresh_per_gt, axis=0)) * is_in_gt.astype(np.float32) ious = overlaps * valid_mask sorted_index = F.argsort(ious, 1) sorted_overlaps = F.gather(ious, 1, sorted_index) max_overlaps = sorted_overlaps[:, :2].flatten() argmax_overlaps = sorted_index[:, :2].flatten() n, c = all_anchors.shape device = all_anchors.device labels = -F.ones(2 * n).to(device) positive_mask = (max_overlaps >= 0.2).to(device).astype(np.float32) negative_mask = (max_overlaps < 0.2).to(device).astype(np.float32) labels = positive_mask + labels * (1 - positive_mask) * (1 - negative_mask) bbox_targets = gt_boxes[argmax_overlaps, :4] all_anchors = F.broadcast_to(F.expand_dims(all_anchors, axis=1), (n, 2, c)).reshape(-1, c) bbox_targets = bbox_transform_opr(all_anchors[:, :4], bbox_targets) labels_cat = gt_boxes[argmax_overlaps, 4] labels_cat = labels_cat * (1 - F.equal(labels, -1).astype( np.float32)) - F.equal(labels, -1).astype(np.float32) return labels, bbox_targets, labels_cat
def rpn_anchor_target_opr_impl(gt_boxes, im_info, anchors, clobber_positives=True, ignore_label=-1, background_label=0): gt_boxes, im_info = gt_boxes.detach(), im_info.detach() anchors = anchors.detach() # NOTE: For multi-gpu version, this function should be re-written a_shp0 = anchors.shape[0] valid_gt_boxes = gt_boxes[:im_info[5], :] valid_mask = (gt_boxes[:im_info[5], 4] > 0).astype(np.float32) overlaps = box_overlap_opr(anchors[:, :4], valid_gt_boxes[:, :4]) overlaps = overlaps * valid_mask.unsqueeze(0) argmax_overlaps = torch.argmax(overlaps, axis=1) max_overlaps = torch.gather(overlaps, 1, argmax_overlaps.unsqueeze(1)) gt_argmax_overlaps = torch.argmax(overlaps, axis=0) gt_argmax_overlaps = torch.gather(overlaps, 1, gt_argmax_overlaps.unsqueeze(0)) cond_max_overlaps = overlaps.eq(gt_argmax_overlaps).astype(np.float32) cmo_shape1 = cond_max_overlaps.shape[1] gt_argmax_overlaps = torch.nonzero(cond_max_overlaps.flatten(), as_tuple=False) gt_argmax_overlaps = gt_argmax_overlaps // cmo_shape1 labels = ignore_label * F.ones(a_shp0) fg_mask = (max_overlaps >= config.rpn_positive_overlap).astype(np.float32) fg_mask[gt_argmax_overlaps] = 1 index = torch.nonzero(fg_mask, as_tuple=False).reshape(-1).long() labels[index] = 1 bbox_targets = bbox_transform_opr(anchors, valid_gt_boxes[index, :4]) # fg_mask[gt_argmax_overlaps] # --- megbrain fashion code --- # argmax_overlaps = O.Argmax(overlaps, axis=1) # max_overlaps = O.IndexingOneHot(overlaps, 1, argmax_overlaps) # gt_argmax_overlaps = O.Argmax(overlaps, axis=0) # gt_max_overlaps = O.IndexingOneHot(overlaps, 0, gt_argmax_overlaps) # cond_max_overlaps = overlaps.eq(gt_max_overlaps.add_axis(0)) # cmo_shape1 = cond_max_overlaps.shape[1] # gt_argmax_overlaps = \ # O.CondTake(cond_max_overlaps.flatten(), cond_max_overlaps.flatten(), # 'EQ',1).outputs[1] # # why should be divided by the cmo_shape1 # gt_argmax_overlaps = gt_argmax_overlaps // cmo_shape1 # labels = O.ones(a_shp0) * ignore_label # const_one = O.ConstProvider(1.0) # if not clobber_positives: # labels = labels * (max_overlaps >= config.rpn_negative_overlap) # fg_mask = (max_overlaps >= config.rpn_positive_overlap) # fg_mask = fg_mask.set_ai[gt_argmax_overlaps]( # const_one.broadcast(gt_argmax_overlaps.shape)) # fg_mask_ind = O.CondTake(fg_mask, fg_mask, 'EQ', 1).outputs[1] # labels = labels.set_ai[fg_mask_ind](const_one.broadcast(fg_mask_ind.shape)) # if clobber_positives: # labels = labels * (max_overlaps >= config.rpn_negative_overlap) # Here, we compute the targets for each anchors # bbox_targets = bbox_transform_opr( # anchors, valid_gt_boxes.ai[argmax_overlaps, :4]) return labels, bbox_targets
def _anchor_target(gt_boxes, im_info, all_anchors): gt_boxes, im_info = gt_boxes.detach(), im_info.detach() all_anchors = all_anchors.detach() gt_boxes = gt_boxes[:im_info[5], :] valid_mask = 1 - (gt_boxes[:, 4] < 0).astype(np.float32) anchor_centers = _compute_center(all_anchors) gtboxes_centers = _compute_center(gt_boxes) * F.expand_dims(valid_mask, axis=0) N, K = all_anchors.shape[0], gt_boxes.shape[0] # an_centers = anchor_centers.unsqueeze(1).repeat(1, K, 1) an_centers = F.expand_dims(anchor_centers, axis=1) gt_centers = F.expand_dims(gtboxes_centers, axis=0) # gt_centers = gtboxes_centers.unsqueeze(0).repeat(N, 1, 1) distance = F.abs(an_centers - gt_centers) distance = F.sqrt(F.pow(distance, 2).sum(axis=2)) start = 0 end = 5 overlaps = box_overlap_opr(all_anchors[:, :4], gt_boxes[:, :4]) overlaps = overlaps * valid_mask.unsqueeze(0) default_num = 9 ious_list = [] for l in range(start, end): index = torch.nonzero(all_anchors[:, 4].eq(l), as_tuple=False)[:, 0] level_dist = level_dist[index, :].transpose(1, 0) ious = distance[index, :].transpose(1, 0) sorted_index = torch.argsort(ious, 1, descending=False) n = min(default_num, sorted_index.shape[1]) ious = torch.gather(ious, 1, sorted_index[:, :n]).transpose(1, 0) ious_list.append(ious) ious = F.concat(ious_list, axis=0) mean_var = ious.mean(0) std_var = ious.std(0) iou_thresh_per_gt = mean_var + std_var iou_thresh_per_gt = torch.clamp(iou_thresh_per_gt, 0.35) n = iou_thresh_per_gt.shape[0] # limits the anchor centers in the gtboxes N, K = all_anchors.shape[0], gt_boxes.shape[0] anchor_points = an_centers proxies = gt_boxes.unsqueeze(0).repeat(N, 1, 1) l = anchor_points[:, :, 0] - proxies[:, :, 0] r = proxies[:, :, 2] - anchor_points[:, :, 0] t = anchor_points[:, :, 1] - proxies[:, :, 1] b = proxies[:, :, 3] - anchor_points[:, :, 1] is_in_gt = F.stack([l, r, t, b], axis=2) is_in_gt = is_in_gt.min(axis=2) > 0.1 valid_mask = (overlaps >= iou_thresh_per_gt.unsqueeze(0)) * is_in_gt ious = overlaps * valid_mask argmax_overlaps = torch.argmax(ious, axis=1) max_overlaps = torch.gather(ious, 1, argmax_overlaps.unsqueeze(1)) n = all_anchors.shape[0] labels = -F.ones(n) positive_mask = max_overlaps > 0 negative_mask = max_overlaps < config.rpn_negative_overlap labels = positive_mask + labels * (1 - positive_mask) * (1 - negative_mask) bbox_targets = gt_boxes[argmax_overlaps, :4] bbox_targets = bbox_transform_opr(all_anchors[:, :4], bbox_targets) labels_cat = gt_boxes[argmax_overlaps, 4] labels_cat = labels_cat * (1 - labels.eq(0).astype(np.float32)) labels_cat = labels_cat * (1 - labels.eq(-1).astype( np.float32)) - labels.eq(-1).astype(np.float32) return labels, bbox_targets, labels_cat