def _build_single_target(self, anchors, valid_flags, gt_boxes,
gt_class_ids):
"""
获取单张图片中的每个建议框是正例还是负例，以及偏移量
Args
---
anchors: [num_anchors, (y1, x1, y2, x2)] 当前图片中所有的建议框
valid_flags: [num_anchors] 当前图片中每个框是否合法
gt_class_ids: [num_gt_boxes]
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
Returns
---
target_matchs: [num_anchors]
target_deltas: [num_rpn_deltas, (dy, dx, dh, dw)]
"""
# 去除那些四个坐标值都是0的框
gt_boxes, _ = trim_zeros(gt_boxes)
# overlaps: [num_anchors, num_gt_boxes] IoU矩阵
overlaps = geometry.compute_overlaps(anchors, gt_boxes)
# 1. 获取建议框标记
# 标记为1：与每个GT框IoU最大的建议框标记为1；与某一GT框的IoU大于pos_iou_thr的建议框标记为1
# 标记为-1：与所有的GT框的IoU都小于neg_iou_thr的框标记为-1
# 标记为0：剩余的框标记为0
# 超越上述的规则：非法的框标记为0
anchor_iou_argmax = tf.argmax(
overlaps, axis=1) # [num_of_anchors,] 每个建议框与哪个GT框IoU最大
anchor_iou_max = tf.reduce_max(overlaps,
axis=[1
]) # [num_of_anchors,] 获取相应的IoU数值
# 所有GT框的IoU都小于neg_iou_thr的框，标记为-1，否则标记为0
target_matchs = tf.where(anchor_iou_max < self.neg_iou_thr,
-tf.ones(anchors.shape[0], dtype=tf.int32),
tf.zeros(anchors.shape[0], dtype=tf.int32))
# 在之前的基础上将非法的框都标记为0
target_matchs = tf.where(tf.equal(valid_flags, 1), target_matchs,
tf.zeros(anchors.shape[0], dtype=tf.int32))
# 在之前的基础上，与某GT框的IoU大于pos_iou_thr的框，标记为1
target_matchs = tf.where(anchor_iou_max >= self.pos_iou_thr,
tf.ones(anchors.shape[0], dtype=tf.int32),
target_matchs)
# [num_gt_boxes] 得到与每个GT框IoU最大的建议框，并将其标记为1
gt_iou_argmax = tf.argmax(overlaps, axis=0)
target_matchs = tf.compat.v1.scatter_update(tf.Variable(target_matchs),
gt_iou_argmax, 1)
# 2. 将框的数量减少至self.num_rpn_deltas
# 若正例框数量大于self.num_rpn_deltas * self.positive_fraction，则用随机选取的方式将多余的框标记为0
# 用负例框填补，使得：正例框数量 + 负例框数量 = self.num_rpn_deltas，删除多余负例框的方式仍然是随机选取
# 只需要self.num_rpn_deltas * self.positive_fraction个正例框，如果多出来了，则将多余的删去（随机删除）
ids = tf.where(tf.equal(target_matchs, 1))
ids = tf.squeeze(ids, 1)
extra = ids.shape.as_list()[0] - int(
self.num_rpn_deltas * self.positive_fraction)
if extra > 0:
ids = tf.random.shuffle(ids)[:extra]
target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0)
# 无论正例框的数量是否能达到self.num_rpn_deltas * self.positive_fraction，不足的部分都用负例框填补
# 使得框数量总共为self.num_rpn_deltas
ids = tf.where(tf.equal(target_matchs, -1))
ids = tf.squeeze(ids, 1)
extra = ids.shape.as_list()[0] - (self.num_rpn_deltas - tf.reduce_sum(
tf.cast(tf.equal(target_matchs, 1), tf.int32)))
if extra > 0:
ids = tf.random.shuffle(ids)[:extra]
target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0)
# 3. 对于所有正例，生成偏移量
# 取出标记为1的anchors存放在a中
ids = tf.where(tf.equal(target_matchs, 1))
a = tf.gather_nd(anchors, ids)
# 取出对应的GT框
anchor_idx = tf.gather_nd(anchor_iou_argmax, ids)
gt = tf.gather(gt_boxes, anchor_idx)
# 计算[正例框数量 * [dy, dx, dh, dw]]
target_deltas = transforms.bbox2delta(a, gt, self.target_means,
self.target_stds)
# 应该返回self.num_rpn_deltas个target_deltas，不足的部分进行零填充
padding = tf.maximum(self.num_rpn_deltas - tf.shape(target_deltas)[0],
0)
target_deltas = tf.pad(target_deltas, [(0, padding), (0, 0)])
return target_matchs, target_deltas
def _build_single_target(self, anchors, valid_flags, gt_boxes, gt_class_ids):
'''Compute targets per instance.
Args
---
anchors: [num_anchors, (y1, x1, y2, x2)]
valid_flags: [num_anchors]
gt_class_ids: [num_gt_boxes]
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
Returns
---
target_matchs: [num_anchors]
target_deltas: [num_rpn_deltas, (dy, dx, log(dh), log(dw))]
'''
gt_boxes, _ = trim_zeros(gt_boxes)
target_matchs = tf.zeros(anchors.shape[0], dtype=tf.int32)
# Compute overlaps [num_anchors, num_gt_boxes]
overlaps = geometry.compute_overlaps(anchors, gt_boxes)
# Match anchors to GT Boxes
# If an anchor overlaps a GT box with IoU >= 0.7 then it's positive.
# If an anchor overlaps a GT box with IoU < 0.3 then it's negative.
# Neutral anchors are those that don't match the conditions above,
# and they don't influence the loss function.
# However, don't keep any GT box unmatched (rare, but happens). Instead,
# match it to the closest anchor (even if its max IoU is < 0.3).
neg_values = tf.constant([0, -1])
pos_values = tf.constant([0, 1])
# 1. Set negative anchors first. They get overwritten below if a GT box is
# matched to them.
anchor_iou_argmax = tf.argmax(overlaps, axis=1)
anchor_iou_max = tf.reduce_max(overlaps, reduction_indices=[1])
target_matchs = tf.where(anchor_iou_max < self.neg_iou_thr,
-tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs)
# filter invalid anchors
target_matchs = tf.where(tf.equal(valid_flags, 1),
target_matchs, tf.zeros(anchors.shape[0], dtype=tf.int32))
# 2. Set anchors with high overlap as positive.
target_matchs = tf.where(anchor_iou_max >= self.pos_iou_thr,
tf.ones(anchors.shape[0], dtype=tf.int32), target_matchs)
# 3. Set an anchor for each GT box (regardless of IoU value).
gt_iou_argmax = tf.argmax(overlaps, axis=0)
target_matchs = tf.scatter_update(tf.Variable(target_matchs), gt_iou_argmax, 1)
# Subsample to balance positive and negative anchors
# Don't let positives be more than half the anchors
ids = tf.where(tf.equal(target_matchs, 1))
ids = tf.squeeze(ids, 1)
extra = ids.shape.as_list()[0] - (self.num_rpn_deltas // 2)
if extra > 0:
# Reset the extra ones to neutral
ids = tf.random_shuffle(ids)[:extra]
target_matchs = tf.scatter_update(target_matchs, ids, 0)
# Same for negative proposals
ids = tf.where(tf.equal(target_matchs, -1))
ids = tf.squeeze(ids, 1)
extra = ids.shape.as_list()[0] - (self.num_rpn_deltas -
tf.reduce_sum(tf.cast(tf.equal(target_matchs, 1), tf.int32)))
if extra > 0:
# Rest the extra ones to neutral
ids = tf.random_shuffle(ids)[:extra]
target_matchs = tf.scatter_update(target_matchs, ids, 0)
# For positive anchors, compute shift and scale needed to transform them
# to match the corresponding GT boxes.
ids = tf.where(tf.equal(target_matchs, 1))
a = tf.gather_nd(anchors, ids)
anchor_idx = tf.gather_nd(anchor_iou_argmax, ids)
gt = tf.gather(gt_boxes, anchor_idx)
target_deltas = transforms.bbox2delta(
a, gt, self.target_means, self.target_stds)
padding = tf.maximum(self.num_rpn_deltas - tf.shape(target_deltas)[0], 0)
target_deltas = tf.pad(target_deltas, [(0, padding), (0, 0)])
return target_matchs, target_deltas
def _build_single_target(self, proposals, gt_boxes, gt_class_ids,
img_shape):
'''
Args
---
proposals: [num_proposals, (y1, x1, y2, x2)] in normalized coordinates.
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
gt_class_ids: [num_gt_boxes]
img_shape: np.ndarray. [2]. (img_height, img_width)
Returns
---
rois: [num_rois, (y1, x1, y2, x2)]
target_matchs: [num_positive_rois]
target_deltas: [num_positive_rois, (dy, dx, log(dh), log(dw))]
'''
H, W = img_shape # 1216, 1216
gt_boxes, non_zeros = trim_zeros(
gt_boxes) # [7, 4], remove padded zero boxes
gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros) # [7]
# normalize (y1, x1, y2, x2) => 0~1
gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32)
# [2k, 4] with [7, 4] => [2k, 7] overlop scores
overlaps = geometry.compute_overlaps(proposals, gt_boxes)
anchor_iou_argmax = tf.argmax(
overlaps, axis=1) # get clost gt boxed id for each anchor boxes
roi_iou_max = tf.reduce_max(
overlaps,
axis=1) # get clost gt boxes overlop score for each anchor boxes
# roi_iou_max: [2000],
positive_roi_bool = (roi_iou_max >= self.pos_iou_thr)
positive_indices = tf.where(positive_roi_bool)[:, 0]
# get all positive indices, namely get all pos_anchor indices
negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0]
# get all negative anchor indices
# Subsample ROIs. Aim for 33% positive
# Positive ROIs
positive_count = int(self.num_rcnn_deltas *
self.positive_fraction) # 0.25?
positive_indices = tf.random.shuffle(
positive_indices)[:positive_count] # [256*0.25]=64, at most get 64
positive_count = tf.shape(positive_indices)[0] # 34
# Negative ROIs. Add enough to maintain positive:negative ratio.
r = 1.0 / self.positive_fraction
negative_count = tf.cast(r * tf.cast(positive_count, tf.float32),
tf.int32) - positive_count #102
negative_indices = tf.random.shuffle(
negative_indices)[:negative_count] #[102]
# Gather selected ROIs
positive_rois = tf.gather(proposals, positive_indices) # [34, 4]
negative_rois = tf.gather(proposals, negative_indices) # [102, 4]
# Assign positive ROIs to GT boxes.
positive_overlaps = tf.gather(overlaps, positive_indices) # [34, 7]
roi_gt_box_assignment = tf.argmax(
positive_overlaps,
axis=1) # for each anchor, get its clost gt boxes
roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) # [34, 4]
target_matchs = tf.gather(gt_class_ids, roi_gt_box_assignment) # [34]
# proposal: [34, 4], target: [34, 4]
target_deltas = transforms.bbox2delta(positive_rois, roi_gt_boxes,
self.target_means,
self.target_stds)
# [34, 4] [102, 4]
rois = tf.concat([positive_rois, negative_rois], axis=0)
N = tf.shape(negative_rois)[0] # 102
target_matchs = tf.pad(target_matchs,
[(0, N)]) # [34] padding after with [N]
target_matchs = tf.stop_gradient(target_matchs) # [34+102]
target_deltas = tf.stop_gradient(target_deltas) # [34, 4]
# rois: [34+102, 4]
return rois, target_matchs, target_deltas
def _build_single_target(self, anchors, valid_flags, gt_boxes,
gt_class_ids):
'''Compute targets per instance.
Args
---
anchors: [num_anchors, (y1, x1, y2, x2)]
valid_flags: [num_anchors]
gt_class_ids: [num_gt_boxes]
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
Returns
---
labels: [num_anchors]
label_weights: [num_anchors]
delta_targets: [num_anchors, (dy, dx, log(dh), log(dw))]
delta_weights: [num_anchors, 4]
'''
gt_boxes, _ = trim_zeros(gt_boxes)
labels = -tf.ones(anchors.shape[0], dtype=tf.int32)
# Compute overlaps [num_anchors, num_gt_boxes]
overlaps = geometry.compute_overlaps(anchors, gt_boxes)
# Match anchors to GT Boxes
# If an anchor overlaps a GT box with IoU >= 0.7 then it's positive.
# If an anchor overlaps a GT box with IoU < 0.3 then it's negative.
# Neutral anchors are those that don't match the conditions above,
# and they don't influence the loss function.
# However, don't keep any GT box unmatched (rare, but happens). Instead,
# match it to the closest anchor (even if its max IoU is < 0.3).
# 1. Set negative anchors first. They get overwritten below if a GT box is
# matched to them.
anchor_iou_argmax = tf.argmax(overlaps, axis=1)
anchor_iou_max = tf.reduce_max(overlaps, axis=[1])
labels = tf.where(anchor_iou_max < self.neg_iou_thr,
tf.zeros(anchors.shape[0], dtype=tf.int32), labels)
# Filter invalid anchors
labels = tf.where(tf.equal(valid_flags, 1), labels,
-tf.ones(anchors.shape[0], dtype=tf.int32))
# 2. Set anchors with high overlap as positive.
labels = tf.where(anchor_iou_max >= self.pos_iou_thr,
tf.ones(anchors.shape[0], dtype=tf.int32), labels)
# 3. Set an anchor for each GT box (regardless of IoU value).
gt_iou_argmax = tf.argmax(overlaps, axis=0)
labels = tf.tensor_scatter_nd_update(
labels, tf.reshape(gt_iou_argmax, (-1, 1)),
tf.ones(gt_iou_argmax.shape, dtype=tf.int32))
# Subsample to balance positive and negative anchors
# Don't let positives be more than half the anchors
ids = tf.where(tf.equal(labels, 1))
extra = ids.shape.as_list()[0] - int(
self.num_rpn_deltas * self.positive_fraction)
if extra > 0:
# Reset the extra ones to neutral
ids = tf.random.shuffle(ids)[:extra]
labels = tf.tensor_scatter_nd_update(
labels, ids, -tf.ones(ids.shape[0], dtype=tf.int32))
# Same for negative proposals
ids = tf.where(tf.equal(labels, 0))
extra = ids.shape.as_list()[0] - (self.num_rpn_deltas - tf.reduce_sum(
tf.cast(tf.equal(labels, 1), tf.int32)))
if extra > 0:
# Rest the extra ones to neutral
ids = tf.random.shuffle(ids)[:extra]
labels = tf.tensor_scatter_nd_update(
labels, ids, -tf.ones(ids.shape[0], dtype=tf.int32))
# For positive anchors, compute shift and scale needed to transform them
# to match the corresponding GT boxes.
# Compute box deltas
gt = tf.gather(gt_boxes, anchor_iou_argmax)
delta_targets = transforms.bbox2delta(anchors, gt, self.target_means,
self.target_stds)
# Compute weights
label_weights = tf.zeros((anchors.shape[0], ), dtype=tf.float32)
delta_weights = tf.zeros((anchors.shape[0], ), dtype=tf.float32)
num_bfg = tf.where(tf.greater_equal(labels, 0)).shape[0]
if num_bfg > 0:
label_weights = tf.where(
labels >= 0,
tf.ones(label_weights.shape, dtype=tf.float32) / num_bfg,
label_weights)
delta_weights = tf.where(
labels > 0,
tf.ones(delta_weights.shape, dtype=tf.float32) / num_bfg,
delta_weights)
delta_weights = tf.tile(tf.reshape(delta_weights, (-1, 1)), [1, 4])
return labels, label_weights, delta_targets, delta_weights
def _build_single_target(self, proposals, gt_boxes, gt_class_ids,
img_shape, batch_ind):
'''
Args
---
proposals: [num_proposals, (batch_ind, y1, x1, y2, x2)] in normalized coordinates.
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
gt_class_ids: [num_gt_boxes]
img_shape: np.ndarray. [2]. (img_height, img_width)
batch_ind: int.
Returns
---
rois: [num_rois, (batch_ind, y1, x1, y2, x2)]
labels: [num_rois]
label_weights: [num_rois]
target_deltas: [num_rois, num_classes, (dy, dx, log(dh), log(dw))]
delta_weights: [num_rois, num_classes, 4]
'''
H, W = img_shape
trimmed_proposals, _ = trim_zeros(proposals[:, 1:])
gt_boxes, non_zeros = trim_zeros(gt_boxes)
gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros)
gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32)
overlaps = geometry.compute_overlaps(trimmed_proposals, gt_boxes)
anchor_iou_argmax = tf.argmax(overlaps, axis=1)
roi_iou_max = tf.reduce_max(overlaps, axis=1)
positive_roi_bool = (roi_iou_max >= self.pos_iou_thr)
positive_indices = tf.where(positive_roi_bool)[:, 0]
negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0]
# Subsample ROIs. Aim for 33% positive
# Positive ROIs
positive_count = int(self.num_rcnn_deltas * self.positive_fraction)
positive_indices = tf.random.shuffle(positive_indices)[:positive_count]
positive_count = tf.shape(positive_indices)[0]
# Negative ROIs. Add enough to maintain positive:negative ratio.
r = 1.0 / self.positive_fraction
negative_count = tf.cast(r * tf.cast(positive_count, tf.float32),
tf.int32) - positive_count
negative_indices = tf.random.shuffle(negative_indices)[:negative_count]
# Gather selected ROIs
positive_rois = tf.gather(proposals,
positive_indices) # [num_positive_rois, 5]
negative_rois = tf.gather(proposals,
negative_indices) # [num_negative_rois, 5]
# Assign positive ROIs to GT boxes.
positive_overlaps = tf.gather(overlaps, positive_indices)
roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1)
roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment)
labels = tf.gather(gt_class_ids, roi_gt_box_assignment)
delta_targets = transforms.bbox2delta(positive_rois[:, 1:],
roi_gt_boxes, self.target_means,
self.target_stds)
rois = tf.concat([positive_rois, negative_rois], axis=0)
N = tf.shape(negative_rois)[0]
P = tf.maximum(self.num_rcnn_deltas - tf.shape(rois)[0], 0)
num_bfg = rois.shape[0]
rois = tf.pad(rois, [(0, P), (0, 0)]) # [num_rois, 5]
labels = tf.pad(labels, [(0, N)], constant_values=0)
# fix bugs: labels=-1 can not used in tf.tensor_scatter_nd_update
labels_index = tf.pad(labels, [(0, P)], constant_values=0)
labels = tf.pad(labels, [(0, P)], constant_values=-1) # [num_rois]
delta_targets = tf.pad(delta_targets, [(0, N + P),
(0, 0)]) # [num_rois, 4]
# Compute weights
label_weights = tf.zeros((self.num_rcnn_deltas, ), dtype=tf.float32)
delta_weights = tf.zeros((self.num_rcnn_deltas, ), dtype=tf.float32)
if num_bfg > 0:
label_weights = tf.where(
labels >= 0,
tf.ones((self.num_rcnn_deltas, ), dtype=tf.float32) / num_bfg,
label_weights)
delta_weights = tf.where(
labels > 0,
tf.ones((self.num_rcnn_deltas, ), dtype=tf.float32) / num_bfg,
delta_weights)
delta_weights = tf.tile(tf.reshape(delta_weights, (-1, 1)),
[1, 4]) # [num_rois, 4]
# Organize Box targets and weights
tile_delta_targets = tf.zeros(
(self.num_rcnn_deltas, self.num_classes, 4))
tile_delta_weights = tf.zeros(
(self.num_rcnn_deltas, self.num_classes, 4))
ids = tf.stack(
[tf.range(self.num_rcnn_deltas, dtype=labels.dtype), labels_index],
axis=1)
delta_targets = tf.tensor_scatter_nd_update(
tile_delta_targets, ids,
delta_targets) # [num_rois, self.num_classes, 4]
delta_weights = tf.tensor_scatter_nd_update(
tile_delta_weights, ids,
delta_weights) # [num_rois, self.num_classes, 4]
return rois, labels, label_weights, delta_targets, delta_weights
def _build_single_target(self, proposals, gt_boxes, gt_class_ids,
img_shape, batch_ind):
'''
Args
---
proposals: [num_proposals, (batch_ind, y1, x1, y2, x2)] in normalized coordinates.
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
gt_class_ids: [num_gt_boxes]
img_shape: np.ndarray. [2]. (img_height, img_width)
batch_ind: int.
Returns
---
rois: [num_rois, (batch_ind, y1, x1, y2, x2)]
target_matchs: [num_rois]
target_deltas: [num_rois, (dy, dx, log(dh), log(dw))]
'''
H, W = img_shape
trimmed_proposals, _ = trim_zeros(proposals[:, 1:])
gt_boxes, non_zeros = trim_zeros(gt_boxes)
gt_class_ids = tf.boolean_mask(gt_class_ids, non_zeros)
gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32)
overlaps = geometry.compute_overlaps(trimmed_proposals, gt_boxes)
anchor_iou_argmax = tf.argmax(overlaps, axis=1)
roi_iou_max = tf.reduce_max(overlaps, axis=1)
positive_roi_bool = (roi_iou_max >= self.pos_iou_thr)
positive_indices = tf.where(positive_roi_bool)[:, 0]
negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0]
# Subsample ROIs. Aim for 33% positive
# Positive ROIs
positive_count = int(self.num_rcnn_deltas * self.positive_fraction)
positive_indices = tf.random_shuffle(positive_indices)[:positive_count]
positive_count = tf.shape(positive_indices)[0]
# Negative ROIs. Add enough to maintain positive:negative ratio.
r = 1.0 / self.positive_fraction
negative_count = tf.cast(r * tf.cast(positive_count, tf.float32),
tf.int32) - positive_count
negative_indices = tf.random_shuffle(negative_indices)[:negative_count]
# Gather selected ROIs
positive_rois = tf.gather(proposals, positive_indices)
negative_rois = tf.gather(proposals, negative_indices)
# Assign positive ROIs to GT boxes.
positive_overlaps = tf.gather(overlaps, positive_indices)
roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1)
roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment)
target_matchs = tf.gather(gt_class_ids, roi_gt_box_assignment)
target_deltas = transforms.bbox2delta(positive_rois[:, 1:],
roi_gt_boxes, self.target_means,
self.target_stds)
rois = tf.concat([positive_rois, negative_rois], axis=0)
N = tf.shape(negative_rois)[0]
P = tf.maximum(self.num_rcnn_deltas - tf.shape(rois)[0], 0)
rois = tf.pad(rois, [(0, P), (0, 0)])
target_matchs = tf.pad(target_matchs, [(0, N)])
target_matchs = tf.pad(target_matchs, [(0, P)], constant_values=-1)
target_deltas = tf.pad(target_deltas, [(0, N + P), (0, 0)])
return rois, target_matchs, target_deltas
def _build_single_target(self, anchors, valid_flags, gt_boxes,
gt_class_ids):
'''Compute targets per instance.
Args
---
anchors: [num_anchors, (y1, x1, y2, x2)]
valid_flags: [num_anchors]
gt_class_ids: [num_gt_boxes]
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)]
Returns
---
target_matchs: [num_anchors]
target_deltas: [num_rpn_deltas, (dy, dx, log(dh), log(dw))]
'''
gt_boxes, _ = trim_zeros(
gt_boxes) # remove padded zero boxes, [new_N, 4]
target_matchs = tf.zeros(anchors.shape[0], dtype=tf.int32) # [326393]
# Compute overlaps [num_anchors, num_gt_boxes] 326393 vs 10 => [326393, 10]
overlaps = geometry.compute_overlaps(anchors, gt_boxes)
# Match anchors to GT Boxes
# If an anchor overlaps ANY GT box with IoU >= 0.7 then it's positive.
# If an anchor overlaps ALL GT box with IoU < 0.3 then it's negative.
# Neutral anchors are those that don't match the conditions above,
# and they don't influence the loss function.
# However, don't keep any GT box unmatched (rare, but happens). Instead,
# match it to the closest anchor (even if its max IoU is < 0.3).
neg_values = tf.constant([0, -1])
pos_values = tf.constant([0, 1])
# 1. Set negative anchors first. They get overwritten below if a GT box is
# matched to them. [N_anchors, N_gt_boxes]
anchor_iou_argmax = tf.argmax(
overlaps, axis=1) # [326396] get clost gt boxes for each anchors
anchor_iou_max = tf.reduce_max(
overlaps,
axis=[1]) # [326396] get closet gt boxes's overlap scores
# if an anchor box overlap all GT box with IoU < 0.3, marked as -1 background
target_matchs = tf.where(anchor_iou_max < self.neg_iou_thr,
-tf.ones(anchors.shape[0], dtype=tf.int32),
target_matchs)
# filter invalid anchors
target_matchs = tf.where(tf.equal(valid_flags, 1), target_matchs,
tf.zeros(anchors.shape[0], dtype=tf.int32))
# if an anchor overlap with any GT box with IoU > 0.7, marked as foreground
# 2. Set anchors with high overlap as positive.
target_matchs = tf.where(anchor_iou_max >= self.pos_iou_thr,
tf.ones(anchors.shape[0], dtype=tf.int32),
target_matchs)
# 3. Set an anchor for each GT box (regardless of IoU value).
gt_iou_argmax = tf.argmax(overlaps, axis=0) # [N_gt_boxes]
target_matchs = tf.compat.v1.scatter_update(tf.Variable(target_matchs),
gt_iou_argmax, 1)
# update corresponding value=>1 for GT boxes' closest boxes
# Subsample to balance positive and negative anchors
# Don't let positives be more than half the anchors
ids = tf.where(tf.equal(target_matchs,
1)) # [N_pos_anchors, 1], [15, 1]
ids = tf.squeeze(ids, 1) # [15]
extra = ids.shape.as_list()[0] - int(
self.num_rpn_deltas * self.positive_fraction) # 256*0.5
if extra > 0: # extra means the redundant pos_anchors
# Reset the extra random ones to neutral
ids = tf.random.shuffle(ids)[:extra]
target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0)
# Same for negative proposals
ids = tf.where(tf.equal(target_matchs, -1)) # [213748, 1]
ids = tf.squeeze(ids, 1)
extra = ids.shape.as_list()[0] - (
self.num_rpn_deltas - # 213748 - (256 - num_of_pos_anchors:15)
tf.reduce_sum(tf.cast(tf.equal(target_matchs, 1), tf.int32)))
if extra > 0: # 213507, so many negative anchors!
# Rest the extra ones to neutral
ids = tf.random.shuffle(ids)[:extra]
target_matchs = tf.compat.v1.scatter_update(target_matchs, ids, 0)
# since we only need 256 anchors, and it had better contains half positive anchors, and harlf neg .
# For positive anchors, compute shift and scale needed to transform them
# to match the corresponding GT boxes.
ids = tf.where(tf.equal(target_matchs, 1)) # [15]
a = tf.gather_nd(anchors, ids) # [369303, 4], [15] => [15, 4]
anchor_idx = tf.gather_nd(
anchor_iou_argmax, ids) # closed gt boxes index for 369303 anchors
gt = tf.gather(
gt_boxes, anchor_idx) # get closed gt boxes coordinates for ids=15
# a: [15, 4], postive anchors, gt: [15, 4] closed gt boxes for each anchors=15
target_deltas = transforms.bbox2delta(a, gt, self.target_means,
self.target_stds)
# target_deltas: [15, (dy,dx,logw,logh)]?
padding = tf.maximum(self.num_rpn_deltas - tf.shape(target_deltas)[0],
0) # 256-15
target_deltas = tf.pad(target_deltas,
[(0, padding),
(0, 0)]) #padding to [256,4], last padding 0
return target_matchs, target_deltas
def _build_single_target(self, proposals, gt_boxes, gt_class_ids,
img_shape):
"""在单张图片上建立target
Args
---
proposals: [num_proposals, (y1, x1, y2, x2)] 建议框坐标，坐标为小数形式
gt_boxes: [num_gt_boxes, (y1, x1, y2, x2)] GT框坐标
gt_class_ids: [num_gt_boxes,] GT框类别号
img_shape: [img_height, img_width] 图片尺寸
Returns
---
rois: [num_rois, [y1, x1, y2, x2]] 坐标为小数形式
target_matchs: [num_rois,] 前num_pos_rios个值是正例对应的GT框序号，后num_neg_rios个值是零填充
target_deltas: [num_positive_rois, [dy, dx, dh, dw]]
"""
H, W = img_shape
# 去除零填充框，框的个数 num_gt_boxes -> num_gt_boxes2
gt_boxes, non_zeros = trim_zeros(
gt_boxes
) # gt_boxes: [num_gt_boxes2, 4], non_zeros: [num_gt_boxes,]
gt_class_ids = tf.boolean_mask(
gt_class_ids, non_zeros) # gt_calss_ids: [num_gt_boxes2,]
# 将GT框坐标化为小数形式
gt_boxes = gt_boxes / tf.constant([H, W, H, W], dtype=tf.float32)
# 计算建议框和GT框的IoU矩阵
overlaps = geometry.compute_overlaps(proposals, gt_boxes)
# 根据pos_iou_thr和neg_iou_thr获取所有正例和负例坐标
# 规则为：与任一GT框的IoU大于pos_iou_thr的建议框为正例，与所有GT框的IoU都小于neg_iou_thr的建议框为负例
roi_iou_max = tf.reduce_max(overlaps, axis=1)
positive_roi_bool = (roi_iou_max >= self.pos_iou_thr)
positive_indices = tf.where(positive_roi_bool)[:,
0] # 降维 [N, 1] =>[N,]
negative_indices = tf.where(roi_iou_max < self.neg_iou_thr)[:, 0]
# 确保正例个数和正、负比例
# 规则为：
# 正例个数不大于self.num_rcnn_deltas * self.positive_fraction
# 正例 : 负例 = self.positive_fraction : (1 - self.positive_fraction)
positive_count = int(self.num_rcnn_deltas * self.positive_fraction)
positive_indices = tf.random.shuffle(positive_indices)[:positive_count]
positive_count = tf.shape(positive_indices)[0]
r = 1.0 / self.positive_fraction
negative_count = tf.cast(r * tf.cast(positive_count, tf.float32),
tf.int32) - positive_count
negative_indices = tf.random.shuffle(negative_indices)[:negative_count]
positive_rois = tf.gather(proposals, positive_indices)
negative_rois = tf.gather(proposals, negative_indices)
# 找出上面选择的正例对应的GT框
positive_overlaps = tf.gather(overlaps, positive_indices)
roi_gt_box_assignment = tf.argmax(positive_overlaps, axis=1)
roi_gt_boxes = tf.gather(gt_boxes, roi_gt_box_assignment) # [34, 4]
target_matchs = tf.gather(gt_class_ids, roi_gt_box_assignment) # [34]
# 计算正例与GT框之间的偏移量
target_deltas = transforms.bbox2delta(positive_rois, roi_gt_boxes,
self.target_means,
self.target_stds)
# [正例框+负例框个数 * [y1, x1, y2, x2]]
rois = tf.concat([positive_rois, negative_rois], axis=0)
# 将target_matchs的长度填充到正例框+负例框个数，使用零填充
N = tf.shape(negative_rois)[0]
target_matchs = tf.pad(target_matchs, [(0, N)])
target_matchs = tf.stop_gradient(target_matchs)
target_deltas = tf.stop_gradient(target_deltas)
return rois, target_matchs, target_deltas
