def showLabRpn(batch_imgs, batch_metas, batch_bboxes, batch_labels): generator = anchor_generator.AnchorGenerator(scales=(32, 64, 128, 256, 512), ratios=(0.5, 1, 2), feature_strides=(4, 8, 16, 32, 64)) anchors, valid_flags = generator.generate_pyramid_anchors(batch_metas) # anchors = anchors[0] valid_flags = valid_flags[0] gt_boxes, _ = trim_zeros(batch_bboxes[0]) target_matchs = tf.zeros(anchors.shape[0], dtype=tf.int32) # Compute overlaps [num_anchors, num_gt_boxes] 326393 vs 10 => [326393, 10] overlaps = geometry.compute_overlaps(anchors, gt_boxes) anchor_iou_max = tf.reduce_max( overlaps, axis=[1]) # [326396] get closet gt boxes's overlap scores target_matchs = tf.where(anchor_iou_max > 0.55, True, False) tem = tf.where(anchor_iou_max < 0.01, True, False) print(tf.boolean_mask(anchors, tem).shape) anchors = tf.boolean_mask(anchors, target_matchs) print(anchors.shape) image = batch_imgs[0].numpy() bboxs = anchors.numpy() for i in range(bboxs.shape[0]): bbox = bboxs[i] # *768 # print(bbox) image = cv2.rectangle(image, (int(float(bbox[0])), int(float(bbox[1]))), (int(float(bbox[2])), int(float(bbox[3]))), (1.0, 0, 0), 2) return image
def _build_single_target(self, anchors, valid_flags, gt_boxes, gt_class_ids): """ 获取单张图片中的每个建议框是正例还是负例,以及偏移量 Args --- anchors: [num_anchors, (y1, x1, y2, x2)] 当前图片中所有的建议框 valid_flags: [num_anchors] 当前图片中每个框是否合法 gt_class_ids: [num_gt_boxes] gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] Returns --- target_matchs: [num_anchors] target_deltas: [num_rpn_deltas, (dy, dx, dh, dw)] """ # 去除那些四个坐标值都是0的框 gt_boxes, _ = trim_zeros(gt_boxes) # overlaps: [num_anchors, num_gt_boxes] IoU矩阵 overlaps = geometry.compute_overlaps(anchors, gt_boxes) # 1. 获取建议框标记 # 标记为1:与每个GT框IoU最大的建议框标记为1;与某一GT框的IoU大于pos_iou_thr的建议框标记为1 # 标记为-1:与所有的GT框的IoU都小于neg_iou_thr的框标记为-1 # 标记为0:剩余的框标记为0 # 超越上述的规则:非法的框标记为0 anchor_iou_argmax = tf.argmax( overlaps, axis=1) # [num_of_anchors,] 每个建议框与哪个GT框IoU最大 anchor_iou_max = tf.reduce_max(overlaps, axis=[1 ]) # [num_of_anchors,] 获取相应的IoU数值 # 所有GT框的IoU都小于neg_iou_thr的框,标记为-1,否则标记为0 target_matchs = tf.where(anchor_iou_max < self.neg_iou_thr, -tf.ones(anchors.shape[0], dtype=tf.int32), tf.zeros(anchors.shape[0], dtype=tf.int32)) # 在之前的基础上将非法的框都标记为0 target_matchs = tf.where(tf.equal(valid_flags, 1), target_matchs, tf.zeros(anchors.shape[0], dtype=tf.int32)) # 在之前的基础上,与某GT框的IoU大于pos_iou_thr的框,标记为1 target_matchs = tf.where(anchor_iou_max >= self.pos_iou_thr, tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs) # [num_gt_boxes] 得到与每个GT框IoU最大的建议框,并将其标记为1 gt_iou_argmax = tf.argmax(overlaps, axis=0) target_matchs = tf.compat.v1.scatter_update(tf.Variable(target_matchs), gt_iou_argmax, 1) # 2. 将框的数量减少至self.num_rpn_deltas # 若正例框数量大于self.num_rpn_deltas * self.positive_fraction,则用随机选取的方式将多余的框标记为0 # 用负例框填补,使得:正例框数量 + 负例框数量 = self.num_rpn_deltas,删除多余负例框的方式仍然是随机选取 # 只需要self.num_rpn_deltas * self.positive_fraction个正例框,如果多出来了,则将多余的删去(随机删除) ids = tf.where(tf.equal(target_matchs, 1)) ids = tf.squeeze(ids, 1) extra = ids.shape.as_list()[0] - int( self.num_rpn_deltas * self.positive_fraction) if extra > 0: ids = tf.random.shuffle(ids)[:extra] target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0) # 无论正例框的数量是否能达到self.num_rpn_deltas * self.positive_fraction,不足的部分都用负例框填补 # 使得框数量总共为self.num_rpn_deltas ids = tf.where(tf.equal(target_matchs, -1)) ids = tf.squeeze(ids, 1) extra = ids.shape.as_list()[0] - (self.num_rpn_deltas - tf.reduce_sum( tf.cast(tf.equal(target_matchs, 1), tf.int32))) if extra > 0: ids = tf.random.shuffle(ids)[:extra] target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0) # 3. 对于所有正例,生成偏移量 # 取出标记为1的anchors存放在a中 ids = tf.where(tf.equal(target_matchs, 1)) a = tf.gather_nd(anchors, ids) # 取出对应的GT框 anchor_idx = tf.gather_nd(anchor_iou_argmax, ids) gt = tf.gather(gt_boxes, anchor_idx) # 计算[正例框数量 * [dy, dx, dh, dw]] target_deltas = transforms.bbox2delta(a, gt, self.target_means, self.target_stds) # 应该返回self.num_rpn_deltas个target_deltas,不足的部分进行零填充 padding = tf.maximum(self.num_rpn_deltas - tf.shape(target_deltas)[0], 0) target_deltas = tf.pad(target_deltas, [(0, padding), (0, 0)]) return target_matchs, target_deltas
def _build_single_target(self, proposals, gt_boxes, gt_class_ids, img_shape): ''' Args --- proposals: [num_proposals, (y1, x1, y2, x2)] in normalized coordinates. gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] gt_class_ids: [num_gt_boxes] img_shape: np.ndarray. [2]. (img_height, img_width) Returns --- rois: [num_rois, (y1, x1, y2, x2)] target_matchs: [num_positive_rois] target_deltas: [num_positive_rois, (dy, dx, log(dh), log(dw))] ''' H, W = img_shape # 1216, 1216 gt_boxes, non_zeros = trim_zeros( gt_boxes) # [7, 4], remove padded zero boxes gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros) # [7] # normalize (y1, x1, y2, x2) => 0~1 gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32) # [2k, 4] with [7, 4] => [2k, 7] overlop scores overlaps = geometry.compute_overlaps(proposals, gt_boxes) anchor_iou_argmax = tf.argmax( overlaps, axis=1) # get clost gt boxed id for each anchor boxes roi_iou_max = tf.reduce_max( overlaps, axis=1) # get clost gt boxes overlop score for each anchor boxes # roi_iou_max: [2000], positive_roi_bool = (roi_iou_max >= self.pos_iou_thr) positive_indices = tf.where(positive_roi_bool)[:, 0] # get all positive indices, namely get all pos_anchor indices negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0] # get all negative anchor indices # Subsample ROIs. Aim for 33% positive # Positive ROIs positive_count = int(self.num_rcnn_deltas * self.positive_fraction) # 0.25? positive_indices = tf.random.shuffle( positive_indices)[:positive_count] # [256*0.25]=64, at most get 64 positive_count = tf.shape(positive_indices)[0] # 34 # Negative ROIs. Add enough to maintain positive:negative ratio. r = 1.0 / self.positive_fraction negative_count = tf.cast(r * tf.cast(positive_count, tf.float32), tf.int32) - positive_count #102 negative_indices = tf.random.shuffle( negative_indices)[:negative_count] #[102] # Gather selected ROIs positive_rois = tf.gather(proposals, positive_indices) # [34, 4] negative_rois = tf.gather(proposals, negative_indices) # [102, 4] # Assign positive ROIs to GT boxes. positive_overlaps = tf.gather(overlaps, positive_indices) # [34, 7] roi_gt_box_assignment = tf.argmax( positive_overlaps, axis=1) # for each anchor, get its clost gt boxes roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) # [34, 4] target_matchs = tf.gather(gt_class_ids, roi_gt_box_assignment) # [34] # proposal: [34, 4], target: [34, 4] target_deltas = transforms.bbox2delta(positive_rois, roi_gt_boxes, self.target_means, self.target_stds) # [34, 4] [102, 4] rois = tf.concat([positive_rois, negative_rois], axis=0) N = tf.shape(negative_rois)[0] # 102 target_matchs = tf.pad(target_matchs, [(0, N)]) # [34] padding after with [N] target_matchs = tf.stop_gradient(target_matchs) # [34+102] target_deltas = tf.stop_gradient(target_deltas) # [34, 4] # rois: [34+102, 4] return rois, target_matchs, target_deltas
def _build_single_target(self, anchors, valid_flags, gt_boxes, gt_class_ids): '''Compute targets per instance. Args --- anchors: [num_anchors, (y1, x1, y2, x2)] valid_flags: [num_anchors] gt_class_ids: [num_gt_boxes] gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] Returns --- target_matchs: [num_anchors] target_deltas: [num_rpn_deltas, (dy, dx, log(dh), log(dw))] ''' gt_boxes, _ = trim_zeros(gt_boxes) target_matchs = tf.zeros(anchors.shape[0], dtype=tf.int32) # Compute overlaps [num_anchors, num_gt_boxes] overlaps = geometry.compute_overlaps(anchors, gt_boxes) # Match anchors to GT Boxes # If an anchor overlaps a GT box with IoU >= 0.7 then it's positive. # If an anchor overlaps a GT box with IoU < 0.3 then it's negative. # Neutral anchors are those that don't match the conditions above, # and they don't influence the loss function. # However, don't keep any GT box unmatched (rare, but happens). Instead, # match it to the closest anchor (even if its max IoU is < 0.3). neg_values = tf.constant([0, -1]) pos_values = tf.constant([0, 1]) # 1. Set negative anchors first. They get overwritten below if a GT box is # matched to them. anchor_iou_argmax = tf.argmax(overlaps, axis=1) anchor_iou_max = tf.reduce_max(overlaps, reduction_indices=[1]) target_matchs = tf.where(anchor_iou_max < self.neg_iou_thr, -tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs) # filter invalid anchors target_matchs = tf.where(tf.equal(valid_flags, 1), target_matchs, tf.zeros(anchors.shape[0], dtype=tf.int32)) # 2. Set anchors with high overlap as positive. target_matchs = tf.where(anchor_iou_max >= self.pos_iou_thr, tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs) # 3. Set an anchor for each GT box (regardless of IoU value). gt_iou_argmax = tf.argmax(overlaps, axis=0) target_matchs = tf.scatter_update(tf.Variable(target_matchs), gt_iou_argmax, 1) # Subsample to balance positive and negative anchors # Don't let positives be more than half the anchors ids = tf.where(tf.equal(target_matchs, 1)) ids = tf.squeeze(ids, 1) extra = ids.shape.as_list()[0] - (self.num_rpn_deltas // 2) if extra > 0: # Reset the extra ones to neutral ids = tf.random_shuffle(ids)[:extra] target_matchs = tf.scatter_update(target_matchs, ids, 0) # Same for negative proposals ids = tf.where(tf.equal(target_matchs, -1)) ids = tf.squeeze(ids, 1) extra = ids.shape.as_list()[0] - (self.num_rpn_deltas - tf.reduce_sum(tf.cast(tf.equal(target_matchs, 1), tf.int32))) if extra > 0: # Rest the extra ones to neutral ids = tf.random_shuffle(ids)[:extra] target_matchs = tf.scatter_update(target_matchs, ids, 0) # For positive anchors, compute shift and scale needed to transform them # to match the corresponding GT boxes. ids = tf.where(tf.equal(target_matchs, 1)) a = tf.gather_nd(anchors, ids) anchor_idx = tf.gather_nd(anchor_iou_argmax, ids) gt = tf.gather(gt_boxes, anchor_idx) target_deltas = transforms.bbox2delta( a, gt, self.target_means, self.target_stds) padding = tf.maximum(self.num_rpn_deltas - tf.shape(target_deltas)[0], 0) target_deltas = tf.pad(target_deltas, [(0, padding), (0, 0)]) return target_matchs, target_deltas
def _build_single_target(self, anchors, valid_flags, gt_boxes, gt_class_ids): '''Compute targets per instance. Args --- anchors: [num_anchors, (y1, x1, y2, x2)] valid_flags: [num_anchors] gt_class_ids: [num_gt_boxes] gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] Returns --- labels: [num_anchors] label_weights: [num_anchors] delta_targets: [num_anchors, (dy, dx, log(dh), log(dw))] delta_weights: [num_anchors, 4] ''' gt_boxes, _ = trim_zeros(gt_boxes) labels = -tf.ones(anchors.shape[0], dtype=tf.int32) # Compute overlaps [num_anchors, num_gt_boxes] overlaps = geometry.compute_overlaps(anchors, gt_boxes) # Match anchors to GT Boxes # If an anchor overlaps a GT box with IoU >= 0.7 then it's positive. # If an anchor overlaps a GT box with IoU < 0.3 then it's negative. # Neutral anchors are those that don't match the conditions above, # and they don't influence the loss function. # However, don't keep any GT box unmatched (rare, but happens). Instead, # match it to the closest anchor (even if its max IoU is < 0.3). # 1. Set negative anchors first. They get overwritten below if a GT box is # matched to them. anchor_iou_argmax = tf.argmax(overlaps, axis=1) anchor_iou_max = tf.reduce_max(overlaps, axis=[1]) labels = tf.where(anchor_iou_max < self.neg_iou_thr, tf.zeros(anchors.shape[0], dtype=tf.int32), labels) # Filter invalid anchors labels = tf.where(tf.equal(valid_flags, 1), labels, -tf.ones(anchors.shape[0], dtype=tf.int32)) # 2. Set anchors with high overlap as positive. labels = tf.where(anchor_iou_max >= self.pos_iou_thr, tf.ones(anchors.shape[0], dtype=tf.int32), labels) # 3. Set an anchor for each GT box (regardless of IoU value). gt_iou_argmax = tf.argmax(overlaps, axis=0) labels = tf.tensor_scatter_nd_update( labels, tf.reshape(gt_iou_argmax, (-1, 1)), tf.ones(gt_iou_argmax.shape, dtype=tf.int32)) # Subsample to balance positive and negative anchors # Don't let positives be more than half the anchors ids = tf.where(tf.equal(labels, 1)) extra = ids.shape.as_list()[0] - int( self.num_rpn_deltas * self.positive_fraction) if extra > 0: # Reset the extra ones to neutral ids = tf.random.shuffle(ids)[:extra] labels = tf.tensor_scatter_nd_update( labels, ids, -tf.ones(ids.shape[0], dtype=tf.int32)) # Same for negative proposals ids = tf.where(tf.equal(labels, 0)) extra = ids.shape.as_list()[0] - (self.num_rpn_deltas - tf.reduce_sum( tf.cast(tf.equal(labels, 1), tf.int32))) if extra > 0: # Rest the extra ones to neutral ids = tf.random.shuffle(ids)[:extra] labels = tf.tensor_scatter_nd_update( labels, ids, -tf.ones(ids.shape[0], dtype=tf.int32)) # For positive anchors, compute shift and scale needed to transform them # to match the corresponding GT boxes. # Compute box deltas gt = tf.gather(gt_boxes, anchor_iou_argmax) delta_targets = transforms.bbox2delta(anchors, gt, self.target_means, self.target_stds) # Compute weights label_weights = tf.zeros((anchors.shape[0], ), dtype=tf.float32) delta_weights = tf.zeros((anchors.shape[0], ), dtype=tf.float32) num_bfg = tf.where(tf.greater_equal(labels, 0)).shape[0] if num_bfg > 0: label_weights = tf.where( labels >= 0, tf.ones(label_weights.shape, dtype=tf.float32) / num_bfg, label_weights) delta_weights = tf.where( labels > 0, tf.ones(delta_weights.shape, dtype=tf.float32) / num_bfg, delta_weights) delta_weights = tf.tile(tf.reshape(delta_weights, (-1, 1)), [1, 4]) return labels, label_weights, delta_targets, delta_weights
def _build_single_target(self, proposals, gt_boxes, gt_class_ids, img_shape, batch_ind): ''' Args --- proposals: [num_proposals, (batch_ind, y1, x1, y2, x2)] in normalized coordinates. gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] gt_class_ids: [num_gt_boxes] img_shape: np.ndarray. [2]. (img_height, img_width) batch_ind: int. Returns --- rois: [num_rois, (batch_ind, y1, x1, y2, x2)] labels: [num_rois] label_weights: [num_rois] target_deltas: [num_rois, num_classes, (dy, dx, log(dh), log(dw))] delta_weights: [num_rois, num_classes, 4] ''' H, W = img_shape trimmed_proposals, _ = trim_zeros(proposals[:, 1:]) gt_boxes, non_zeros = trim_zeros(gt_boxes) gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros) gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32) overlaps = geometry.compute_overlaps(trimmed_proposals, gt_boxes) anchor_iou_argmax = tf.argmax(overlaps, axis=1) roi_iou_max = tf.reduce_max(overlaps, axis=1) positive_roi_bool = (roi_iou_max >= self.pos_iou_thr) positive_indices = tf.where(positive_roi_bool)[:, 0] negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0] # Subsample ROIs. Aim for 33% positive # Positive ROIs positive_count = int(self.num_rcnn_deltas * self.positive_fraction) positive_indices = tf.random.shuffle(positive_indices)[:positive_count] positive_count = tf.shape(positive_indices)[0] # Negative ROIs. Add enough to maintain positive:negative ratio. r = 1.0 / self.positive_fraction negative_count = tf.cast(r * tf.cast(positive_count, tf.float32), tf.int32) - positive_count negative_indices = tf.random.shuffle(negative_indices)[:negative_count] # Gather selected ROIs positive_rois = tf.gather(proposals, positive_indices) # [num_positive_rois, 5] negative_rois = tf.gather(proposals, negative_indices) # [num_negative_rois, 5] # Assign positive ROIs to GT boxes. positive_overlaps = tf.gather(overlaps, positive_indices) roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1) roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) labels = tf.gather(gt_class_ids, roi_gt_box_assignment) delta_targets = transforms.bbox2delta(positive_rois[:, 1:], roi_gt_boxes, self.target_means, self.target_stds) rois = tf.concat([positive_rois, negative_rois], axis=0) N = tf.shape(negative_rois)[0] P = tf.maximum(self.num_rcnn_deltas - tf.shape(rois)[0], 0) num_bfg = rois.shape[0] rois = tf.pad(rois, [(0, P), (0, 0)]) # [num_rois, 5] labels = tf.pad(labels, [(0, N)], constant_values=0) # fix bugs: labels=-1 can not used in tf.tensor_scatter_nd_update labels_index = tf.pad(labels, [(0, P)], constant_values=0) labels = tf.pad(labels, [(0, P)], constant_values=-1) # [num_rois] delta_targets = tf.pad(delta_targets, [(0, N + P), (0, 0)]) # [num_rois, 4] # Compute weights label_weights = tf.zeros((self.num_rcnn_deltas, ), dtype=tf.float32) delta_weights = tf.zeros((self.num_rcnn_deltas, ), dtype=tf.float32) if num_bfg > 0: label_weights = tf.where( labels >= 0, tf.ones((self.num_rcnn_deltas, ), dtype=tf.float32) / num_bfg, label_weights) delta_weights = tf.where( labels > 0, tf.ones((self.num_rcnn_deltas, ), dtype=tf.float32) / num_bfg, delta_weights) delta_weights = tf.tile(tf.reshape(delta_weights, (-1, 1)), [1, 4]) # [num_rois, 4] # Organize Box targets and weights tile_delta_targets = tf.zeros( (self.num_rcnn_deltas, self.num_classes, 4)) tile_delta_weights = tf.zeros( (self.num_rcnn_deltas, self.num_classes, 4)) ids = tf.stack( [tf.range(self.num_rcnn_deltas, dtype=labels.dtype), labels_index], axis=1) delta_targets = tf.tensor_scatter_nd_update( tile_delta_targets, ids, delta_targets) # [num_rois, self.num_classes, 4] delta_weights = tf.tensor_scatter_nd_update( tile_delta_weights, ids, delta_weights) # [num_rois, self.num_classes, 4] return rois, labels, label_weights, delta_targets, delta_weights
def _build_single_target(self, proposals, gt_boxes, gt_class_ids, img_shape, batch_ind): ''' Args --- proposals: [num_proposals, (batch_ind, y1, x1, y2, x2)] in normalized coordinates. gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] gt_class_ids: [num_gt_boxes] img_shape: np.ndarray. [2]. (img_height, img_width) batch_ind: int. Returns --- rois: [num_rois, (batch_ind, y1, x1, y2, x2)] target_matchs: [num_rois] target_deltas: [num_rois, (dy, dx, log(dh), log(dw))] ''' H, W = img_shape trimmed_proposals, _ = trim_zeros(proposals[:, 1:]) gt_boxes, non_zeros = trim_zeros(gt_boxes) gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros) gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32) overlaps = geometry.compute_overlaps(trimmed_proposals, gt_boxes) anchor_iou_argmax = tf.argmax(overlaps, axis=1) roi_iou_max = tf.reduce_max(overlaps, axis=1) positive_roi_bool = (roi_iou_max >= self.pos_iou_thr) positive_indices = tf.where(positive_roi_bool)[:, 0] negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0] # Subsample ROIs. Aim for 33% positive # Positive ROIs positive_count = int(self.num_rcnn_deltas * self.positive_fraction) positive_indices = tf.random_shuffle(positive_indices)[:positive_count] positive_count = tf.shape(positive_indices)[0] # Negative ROIs. Add enough to maintain positive:negative ratio. r = 1.0 / self.positive_fraction negative_count = tf.cast(r * tf.cast(positive_count, tf.float32), tf.int32) - positive_count negative_indices = tf.random_shuffle(negative_indices)[:negative_count] # Gather selected ROIs positive_rois = tf.gather(proposals, positive_indices) negative_rois = tf.gather(proposals, negative_indices) # Assign positive ROIs to GT boxes. positive_overlaps = tf.gather(overlaps, positive_indices) roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1) roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) target_matchs = tf.gather(gt_class_ids, roi_gt_box_assignment) target_deltas = transforms.bbox2delta(positive_rois[:, 1:], roi_gt_boxes, self.target_means, self.target_stds) rois = tf.concat([positive_rois, negative_rois], axis=0) N = tf.shape(negative_rois)[0] P = tf.maximum(self.num_rcnn_deltas - tf.shape(rois)[0], 0) rois = tf.pad(rois, [(0, P), (0, 0)]) target_matchs = tf.pad(target_matchs, [(0, N)]) target_matchs = tf.pad(target_matchs, [(0, P)], constant_values=-1) target_deltas = tf.pad(target_deltas, [(0, N + P), (0, 0)]) return rois, target_matchs, target_deltas
def _build_single_target(self, anchors, valid_flags, gt_boxes, gt_class_ids): '''Compute targets per instance. Args --- anchors: [num_anchors, (y1, x1, y2, x2)] valid_flags: [num_anchors] gt_class_ids: [num_gt_boxes] gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] Returns --- target_matchs: [num_anchors] target_deltas: [num_rpn_deltas, (dy, dx, log(dh), log(dw))] ''' gt_boxes, _ = trim_zeros( gt_boxes) # remove padded zero boxes, [new_N, 4] target_matchs = tf.zeros(anchors.shape[0], dtype=tf.int32) # [326393] # Compute overlaps [num_anchors, num_gt_boxes] 326393 vs 10 => [326393, 10] overlaps = geometry.compute_overlaps(anchors, gt_boxes) # Match anchors to GT Boxes # If an anchor overlaps ANY GT box with IoU >= 0.7 then it's positive. # If an anchor overlaps ALL GT box with IoU < 0.3 then it's negative. # Neutral anchors are those that don't match the conditions above, # and they don't influence the loss function. # However, don't keep any GT box unmatched (rare, but happens). Instead, # match it to the closest anchor (even if its max IoU is < 0.3). neg_values = tf.constant([0, -1]) pos_values = tf.constant([0, 1]) # 1. Set negative anchors first. They get overwritten below if a GT box is # matched to them. [N_anchors, N_gt_boxes] anchor_iou_argmax = tf.argmax( overlaps, axis=1) # [326396] get clost gt boxes for each anchors anchor_iou_max = tf.reduce_max( overlaps, axis=[1]) # [326396] get closet gt boxes's overlap scores # if an anchor box overlap all GT box with IoU < 0.3, marked as -1 background target_matchs = tf.where(anchor_iou_max < self.neg_iou_thr, -tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs) # filter invalid anchors target_matchs = tf.where(tf.equal(valid_flags, 1), target_matchs, tf.zeros(anchors.shape[0], dtype=tf.int32)) # if an anchor overlap with any GT box with IoU > 0.7, marked as foreground # 2. Set anchors with high overlap as positive. target_matchs = tf.where(anchor_iou_max >= self.pos_iou_thr, tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs) # 3. Set an anchor for each GT box (regardless of IoU value). gt_iou_argmax = tf.argmax(overlaps, axis=0) # [N_gt_boxes] target_matchs = tf.compat.v1.scatter_update(tf.Variable(target_matchs), gt_iou_argmax, 1) # update corresponding value=>1 for GT boxes' closest boxes # Subsample to balance positive and negative anchors # Don't let positives be more than half the anchors ids = tf.where(tf.equal(target_matchs, 1)) # [N_pos_anchors, 1], [15, 1] ids = tf.squeeze(ids, 1) # [15] extra = ids.shape.as_list()[0] - int( self.num_rpn_deltas * self.positive_fraction) # 256*0.5 if extra > 0: # extra means the redundant pos_anchors # Reset the extra random ones to neutral ids = tf.random.shuffle(ids)[:extra] target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0) # Same for negative proposals ids = tf.where(tf.equal(target_matchs, -1)) # [213748, 1] ids = tf.squeeze(ids, 1) extra = ids.shape.as_list()[0] - ( self.num_rpn_deltas - # 213748 - (256 - num_of_pos_anchors:15) tf.reduce_sum(tf.cast(tf.equal(target_matchs, 1), tf.int32))) if extra > 0: # 213507, so many negative anchors! # Rest the extra ones to neutral ids = tf.random.shuffle(ids)[:extra] target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0) # since we only need 256 anchors, and it had better contains half positive anchors, and harlf neg . # For positive anchors, compute shift and scale needed to transform them # to match the corresponding GT boxes. ids = tf.where(tf.equal(target_matchs, 1)) # [15] a = tf.gather_nd(anchors, ids) # [369303, 4], [15] => [15, 4] anchor_idx = tf.gather_nd( anchor_iou_argmax, ids) # closed gt boxes index for 369303 anchors gt = tf.gather( gt_boxes, anchor_idx) # get closed gt boxes coordinates for ids=15 # a: [15, 4], postive anchors, gt: [15, 4] closed gt boxes for each anchors=15 target_deltas = transforms.bbox2delta(a, gt, self.target_means, self.target_stds) # target_deltas: [15, (dy,dx,logw,logh)]? padding = tf.maximum(self.num_rpn_deltas - tf.shape(target_deltas)[0], 0) # 256-15 target_deltas = tf.pad(target_deltas, [(0, padding), (0, 0)]) #padding to [256,4], last padding 0 return target_matchs, target_deltas
def _build_single_target(self, proposals, gt_boxes, gt_class_ids, img_shape): """在单张图片上建立target Args --- proposals: [num_proposals, (y1, x1, y2, x2)] 建议框坐标,坐标为小数形式 gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] GT框坐标 gt_class_ids: [num_gt_boxes,] GT框类别号 img_shape: [img_height, img_width] 图片尺寸 Returns --- rois: [num_rois, [y1, x1, y2, x2]] 坐标为小数形式 target_matchs: [num_rois,] 前num_pos_rios个值是正例对应的GT框序号,后num_neg_rios个值是零填充 target_deltas: [num_positive_rois, [dy, dx, dh, dw]] """ H, W = img_shape # 去除零填充框,框的个数 num_gt_boxes -> num_gt_boxes2 gt_boxes, non_zeros = trim_zeros( gt_boxes ) # gt_boxes: [num_gt_boxes2, 4], non_zeros: [num_gt_boxes,] gt_class_ids = tf.boolean_mask( gt_class_ids, non_zeros) # gt_calss_ids: [num_gt_boxes2,] # 将GT框坐标化为小数形式 gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32) # 计算建议框和GT框的IoU矩阵 overlaps = geometry.compute_overlaps(proposals, gt_boxes) # 根据pos_iou_thr和neg_iou_thr获取所有正例和负例坐标 # 规则为:与任一GT框的IoU大于pos_iou_thr的建议框为正例,与所有GT框的IoU都小于neg_iou_thr的建议框为负例 roi_iou_max = tf.reduce_max(overlaps, axis=1) positive_roi_bool = (roi_iou_max >= self.pos_iou_thr) positive_indices = tf.where(positive_roi_bool)[:, 0] # 降维 [N, 1] =>[N,] negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0] # 确保正例个数和正、负比例 # 规则为: # 正例个数不大于self.num_rcnn_deltas * self.positive_fraction # 正例 : 负例 = self.positive_fraction : (1 - self.positive_fraction) positive_count = int(self.num_rcnn_deltas * self.positive_fraction) positive_indices = tf.random.shuffle(positive_indices)[:positive_count] positive_count = tf.shape(positive_indices)[0] r = 1.0 / self.positive_fraction negative_count = tf.cast(r * tf.cast(positive_count, tf.float32), tf.int32) - positive_count negative_indices = tf.random.shuffle(negative_indices)[:negative_count] positive_rois = tf.gather(proposals, positive_indices) negative_rois = tf.gather(proposals, negative_indices) # 找出上面选择的正例对应的GT框 positive_overlaps = tf.gather(overlaps, positive_indices) roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1) roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) # [34, 4] target_matchs = tf.gather(gt_class_ids, roi_gt_box_assignment) # [34] # 计算正例与GT框之间的偏移量 target_deltas = transforms.bbox2delta(positive_rois, roi_gt_boxes, self.target_means, self.target_stds) # [正例框+负例框个数 * [y1, x1, y2, x2]] rois = tf.concat([positive_rois, negative_rois], axis=0) # 将target_matchs的长度填充到正例框+负例框个数,使用零填充 N = tf.shape(negative_rois)[0] target_matchs = tf.pad(target_matchs, [(0, N)]) target_matchs = tf.stop_gradient(target_matchs) target_deltas = tf.stop_gradient(target_deltas) return rois, target_matchs, target_deltas