def detect_targets_graph(gt_boxes, gt_class_ids, proposals,
train_rois_per_image, roi_positive_ratio):
"""
每个图像生成检测网络的分类和回归目标
IoU>=0.5的为正样本；IoU<0.5的为负样本
:param gt_boxes: GT 边框坐标 [MAX_GT_BOXs, (y1,x1,y2,x2,tag)] ,tag=0 为padding
:param gt_class_ids: GT 类别 [MAX_GT_BOXs, 1+1] ;最后一位为tag, tag=0 为padding
:param proposals: [N,(y1,x1,y2,x2,tag)] ,tag=0 为padding
:param train_rois_per_image: 每张图像训练的proposal数量
:param roi_positive_ratio: proposal正负样本比
:return:
"""
# 去除padding
gt_boxes = tf_utils.remove_pad(gt_boxes)
gt_class_ids = tf_utils.remove_pad(gt_class_ids)[:, 0] # 从[N,1]变为[N]
proposals = tf_utils.remove_pad(proposals)
proposals_num = tf.shape(proposals)[0]
# 计算iou
iou = compute_iou(gt_boxes, proposals) # [gt_num,rois_num]
# iou >=0.5为正
proposals_iou_max = tf.reduce_max(iou, axis=0) # [rois_num]
positive_indices = tf.where(
tf.logical_and(tf.equal(iou, proposals_iou_max),
tf.greater_equal(iou, 0.5)))
gt_pos_idx = positive_indices[:, 0] # 第一维gt索引
proposal_pos_idx = positive_indices[:, 1] # 第二位rois索引
match_gt_num = tf.shape(tf.unique(gt_pos_idx)[0])[0] # shuffle 前匹配的gt num
gt_boxes_pos = tf.gather(gt_boxes, gt_pos_idx)
class_ids = tf.gather(gt_class_ids, gt_pos_idx)
proposal_pos = tf.gather(proposals, proposal_pos_idx)
# 根据正负样本比确定最终的正样本
positive_num = tf.minimum(
tf.shape(proposal_pos)[0],
int(train_rois_per_image * roi_positive_ratio))
gt_boxes_pos, class_ids, proposal_pos, gt_pos_idx = shuffle_sample(
[gt_boxes_pos, class_ids, proposal_pos, gt_pos_idx],
tf.shape(proposal_pos)[0], positive_num)
match_gt_num_after_shuffle = tf.shape(
tf.unique(gt_pos_idx)[0])[0] # shuffle 后匹配的gt num
# 计算回归目标
deltas = regress_target(proposal_pos, gt_boxes_pos)
# 负样本：与所有GT的iou<0.5且iou>0.1
proposal_iou_max = tf.reduce_max(iou, axis=0)
proposal_neg_idx = tf.cond( # 需要考虑GT个数为0的情况;全部都是负样本
tf.greater(tf.shape(gt_boxes)[0], 0),
true_fn=lambda: tf.where(
tf.logical_and(proposal_iou_max < 0.5, proposal_iou_max > 0.1))[:,
0],
false_fn=lambda: tf.cast(tf.range(proposals_num), dtype=tf.int64))
# 确定负样本数量
negative_num = tf.minimum(train_rois_per_image - positive_num,
tf.shape(proposal_neg_idx)[0])
proposal_neg_idx = tf.random_shuffle(proposal_neg_idx)[:negative_num]
# 收集负样本
proposal_neg = tf.gather(proposals, proposal_neg_idx)
class_ids_neg = tf.zeros(shape=[negative_num]) # 背景类，类别id为0
deltas_neg = tf.zeros(shape=[negative_num, 4])
# 合并正负样本
train_rois = tf.concat([proposal_pos, proposal_neg], axis=0)
deltas = tf.concat([deltas, deltas_neg], axis=0)
class_ids = tf.concat([class_ids, class_ids_neg], axis=0)
# 计算padding
class_ids, train_rois = tf_utils.pad_list_to_fixed_size(
[tf.expand_dims(class_ids, axis=1), train_rois],
train_rois_per_image) # class_ids分类扩一维
# 为后续处理方便负样本tag设置为-1
deltas = tf_utils.pad_to_fixed_size_with_negative(
deltas, train_rois_per_image, negative_num=negative_num)
# 其它统计指标
gt_num = tf.shape(gt_class_ids)[0] # GT数
miss_gt_num = gt_num - match_gt_num
miss_gt_num_shuffle = gt_num - match_gt_num_after_shuffle # shuffle后未分配roi的GT
gt_min_max_iou = tf.reduce_min(tf.reduce_max(iou, axis=1)) # gt 匹配最小最大值
train_rois.set_shape([train_rois_per_image, 5])
return [
deltas, class_ids, train_rois,
tf_utils.scalar_to_1d_tensor(miss_gt_num),
tf_utils.scalar_to_1d_tensor(miss_gt_num_shuffle),
tf_utils.scalar_to_1d_tensor(gt_min_max_iou),
tf_utils.scalar_to_1d_tensor(positive_num),
tf_utils.scalar_to_1d_tensor(negative_num),
tf_utils.scalar_to_1d_tensor(proposals_num)
]
def rpn_targets_graph(gt_boxes,
gt_cls,
anchors,
anchors_tag,
rpn_train_anchors=None):
"""
处理单个图像的rpn分类和回归目标
a)正样本为 IoU>0.7的anchor;负样本为IoU<0.3的anchor; 居中的为中性样本，丢弃
b)需要保证所有的GT都有anchor对应，即使IoU<0.3；
c)正负样本比例保持1:1
:param gt_boxes: GT 边框坐标 [MAX_GT_BOXs, (y1,x1,y2,x2,tag)] ,tag=0 为padding
:param gt_cls: GT 类别 [MAX_GT_BOXs, 1+1] ;最后一位为tag, tag=0 为padding
:param anchors: [anchor_num, (y1,x1,y2,x2)]
:param anchors_tag:[anchor_num] bool类型
:param rpn_train_anchors: 训练样本数(256)
:return:
deltas:[rpn_train_anchors,(dy,dx,dh,dw,tag)]：anchor边框回归目标,tag=1 为正样本，tag=0为padding，tag=-1为负样本
class_ids:[rpn_train_anchors,1+1]: anchor边框分类,tag=1 为正样本，tag=0为padding，tag=-1为负样本
indices:[rpn_train_anchors,(indices,tag)]: tag=1 为正样本，tag=0为padding，tag=-1为负样本
"""
# 获取真正的GT,去除标签位
gt_boxes = tf_utils.remove_pad(gt_boxes)
gt_cls = tf_utils.remove_pad(gt_cls)[:, 0] # [N,1]转[N]
# 获取有效的anchors
valid_anchor_indices = tf.where(anchors_tag)[:, 0] # [valid_anchors_num]
anchors = tf.gather(anchors, valid_anchor_indices)
# 计算IoU
iou = compute_iou(gt_boxes, anchors)
# print("iou:{}".format(iou))
# 每个GT对应的IoU最大的anchor是正样本
gt_iou_argmax = tf.argmax(iou, axis=1)
positive_gt_indices_1 = tf.range(tf.shape(gt_boxes)[0]) # 索引号就是1..n-1
positive_anchor_indices_1 = gt_iou_argmax
# 每个anchors最大iou ，且iou>0.7的为正样本
anchors_iou_max = tf.reduce_max(iou, axis=0)
# 正样本索引号（iou>0.7),
positive_anchor_indices_2 = tf.where(
anchors_iou_max > 0.7, name='rpn_target_positive_indices') # [:, 0]
# 找到正样本对应的GT boxes 索引
# anchors_iou_argmax = tf.argmax(iou, axis=0) # 每个anchor最大iou对应的GT 索引 [n]
anchors_iou_argmax = tf.cond( # 需要考虑GT个数为0的情况
tf.greater(tf.shape(gt_boxes)[0], 0),
true_fn=lambda: tf.argmax(iou, axis=0),
false_fn=lambda: tf.cast(tf.constant([]), tf.int64))
positive_gt_indices_2 = tf.gather_nd(anchors_iou_argmax,
positive_anchor_indices_2)
# 合并两部分正样本
positive_gt_indices = tf.concat(
[positive_gt_indices_1,
tf.cast(positive_gt_indices_2, tf.int32)],
axis=0,
name='rpn_gt_boxes_concat')
positive_anchor_indices = tf.concat(
[positive_anchor_indices_1, positive_anchor_indices_2[:, 0]],
axis=0,
name='rpn_positive_anchors_concat')
# 根据正负样本比1:1,确定最终的正样本
positive_num = tf.minimum(
tf.shape(positive_anchor_indices)[0], int(rpn_train_anchors * 0.9))
positive_anchor_indices, positive_gt_indices = shuffle_sample(
[positive_anchor_indices, positive_gt_indices],
tf.shape(positive_anchor_indices)[0], positive_num)
# 根据索引选择anchor和GT
positive_anchors = tf.gather(anchors, positive_anchor_indices)
positive_gt_boxes = tf.gather(gt_boxes, positive_gt_indices)
positive_gt_cls = tf.gather(gt_cls, positive_gt_indices)
# 回归目标计算
deltas = regress_target(positive_anchors, positive_gt_boxes)
# 处理负样本
negative_indices = tf.where(anchors_iou_max < 0.3,
name='rpn_target_negative_indices') # [:, 0]
# 负样本,保证负样本不超过一半
negative_num = tf.minimum(rpn_train_anchors - positive_num,
tf.shape(negative_indices)[0],
name='rpn_target_negative_num')
# negative_num = tf.minimum(int(rpn_train_anchors * 0.5), negative_num, name='rpn_target_negative_num_2')
negative_indices = tf.random_shuffle(negative_indices)[:negative_num]
negative_gt_cls = tf.zeros([negative_num]) # 负样本类别id为0
negative_deltas = tf.zeros([negative_num, 4])
# 合并正负样本
deltas = tf.concat([deltas, negative_deltas],
axis=0,
name='rpn_target_deltas')
class_ids = tf.concat([positive_gt_cls, negative_gt_cls],
axis=0,
name='rpn_target_class_ids')
indices = tf.concat([positive_anchor_indices, negative_indices[:, 0]],
axis=0,
name='rpn_train_anchor_indices')
# indices转换会原始的anchors索引
indices = tf.gather(valid_anchor_indices,
indices,
name='map_to_origin_anchor_indices')
# 计算padding
deltas, class_ids = tf_utils.pad_list_to_fixed_size(
[deltas, tf.expand_dims(class_ids, 1)], rpn_train_anchors)
# 将负样本tag标志改为-1;方便后续处理;
indices = tf_utils.pad_to_fixed_size_with_negative(
tf.expand_dims(indices, 1),
rpn_train_anchors,
negative_num=negative_num,
data_type=tf.int64)
# 其它统计指标
gt_num = tf.shape(gt_cls)[0] # GT数
miss_match_gt_num = gt_num - tf.shape(
tf.unique(positive_gt_indices)[0])[0] # 未分配anchor的GT
rpn_gt_min_max_iou = tf.reduce_min(tf.reduce_max(
iou, axis=1)) # GT匹配anchor最小的IoU
return [
deltas, class_ids, indices,
tf_utils.scalar_to_1d_tensor(gt_num),
tf_utils.scalar_to_1d_tensor(positive_num),
tf_utils.scalar_to_1d_tensor(negative_num),
tf_utils.scalar_to_1d_tensor(miss_match_gt_num),
tf_utils.scalar_to_1d_tensor(rpn_gt_min_max_iou)
]
def detect_targets_graph(gt_boxes, gt_class_ids, proposals,
train_rois_per_image, roi_positive_ratio):
"""
每个图像生成检测网络的分类和回归目标
IoU>=0.5的为正样本；IoU<0.5的为负样本
:param gt_boxes: GT 边框坐标 [MAX_GT_BOXs, (y1,x1,y2,x2,tag)] ,tag=0 为padding
:param gt_class_ids: GT 类别 [MAX_GT_BOXs, 1+1] ;最后一位为tag, tag=0 为padding
:param proposals: [N,(y1,x1,y2,x2,tag)] ,tag=0 为padding
:param train_rois_per_image: 每张图像训练的proposal数量
:param roi_positive_ratio: proposal正负样本比
:return:
"""
# 去除padding
gt_boxes = tf_utils.remove_pad(gt_boxes)
gt_class_ids = tf_utils.remove_pad(gt_class_ids)[:, 0] # 从[N,1]变为[N]
proposals = tf_utils.remove_pad(proposals)
# 计算iou
iou = compute_iou(gt_boxes, proposals)
# 每个GT边框IoU最大的proposal为正
# gt_iou_argmax = tf.argmax(iou, axis=1)
gt_iou_argmax = tf.cond( # 需要考虑proposal个数为0的情况
tf.greater(tf.shape(proposals)[0], 0),
true_fn=lambda: tf.argmax(iou, axis=1),
false_fn=lambda: tf.cast(tf.constant([]), tf.int64))
# GT和对应的proposal
# gt_boxes_pos_1 = tf.identity(gt_boxes)
# gt_class_ids_pos_1 = tf.identity(gt_class_ids)
# proposal_pos_1 = tf.gather(proposals, gt_iou_argmax)
# 在接下来的操作之前提出已经被选中的proposal
indices = tf.unique(gt_iou_argmax)[0] # 被选中的索引
all_indices = tf.range(tf.shape(proposals)[0]) # 所有的索引
remainder_indices = tf.setdiff1d(all_indices,
tf.cast(indices, tf.int32))[0] # 剩余的索引
# 剩余的proposals和iou
proposals = tf.gather(proposals, remainder_indices)
iou = tf.gather(iou, remainder_indices, axis=1)
# 正样本每个proposal 最大的iou,且iou>=0.5
proposal_iou_max = tf.reduce_max(iou, axis=0)
proposal_pos_idx = tf.where(
proposal_iou_max >= 0.5) # 正样本proposal对应的索引号,二维
# proposal_iou_argmax = tf.argmax(iou, axis=0)
proposal_iou_argmax = tf.cond( # 需要考虑GT个数为0的情况
tf.greater(tf.shape(gt_boxes)[0], 0),
true_fn=lambda: tf.argmax(iou, axis=0),
false_fn=lambda: tf.cast(tf.constant([]), tf.int64))
gt_pos_idx = tf.gather_nd(proposal_iou_argmax,
proposal_pos_idx) # 对应的GT 索引号，一维的
gt_boxes_pos_2 = tf.gather(gt_boxes, gt_pos_idx)
gt_class_ids_pos_2 = tf.gather(gt_class_ids, gt_pos_idx)
proposal_pos_2 = tf.gather_nd(proposals, proposal_pos_idx)
# 合并两部分正样本
# gt_boxes_pos = tf.concat([gt_boxes_pos_1, gt_boxes_pos_2], axis=0)
# class_ids = tf.concat([gt_class_ids_pos_1, gt_class_ids_pos_2], axis=0)
# proposal_pos = tf.concat([proposal_pos_1, proposal_pos_2], axis=0)
gt_boxes_pos = gt_boxes_pos_2
class_ids = gt_class_ids_pos_2
proposal_pos = proposal_pos_2
# 根据正负样本比确定最终的正样本
positive_num = tf.minimum(
tf.shape(proposal_pos)[0],
int(train_rois_per_image * roi_positive_ratio))
gt_boxes_pos, class_ids, proposal_pos = shuffle_sample(
[gt_boxes_pos, class_ids, proposal_pos],
tf.shape(proposal_pos)[0], positive_num)
# 计算回归目标
deltas = regress_target(proposal_pos, gt_boxes_pos)
# 负样本：与所有GT的iou<0.5
proposal_neg_idx = tf.where(proposal_iou_max < 0.5)
# 确定负样本数量
negative_num = tf.minimum(train_rois_per_image - positive_num,
tf.shape(proposal_neg_idx)[0])
proposal_neg_idx = tf.random_shuffle(proposal_neg_idx)[:negative_num]
# 收集负样本
proposal_neg = tf.gather_nd(proposals, proposal_neg_idx)
class_ids_neg = tf.zeros(shape=[negative_num]) # 背景类，类别id为0
deltas_neg = tf.zeros(shape=[negative_num, 4])
# 合并正负样本
train_rois = tf.concat([proposal_pos, proposal_neg], axis=0)
deltas = tf.concat([deltas, deltas_neg], axis=0)
class_ids = tf.concat([class_ids, class_ids_neg], axis=0)
# 计算padding
class_ids, train_rois = tf_utils.pad_list_to_fixed_size(
[tf.expand_dims(class_ids, axis=1), train_rois],
train_rois_per_image) # class_ids分类扩一维
# 为后续处理方便负样本tag设置为-1
deltas = tf_utils.pad_to_fixed_size_with_negative(
deltas, train_rois_per_image, negative_num=negative_num)
# 其它统计指标
gt_num = tf.shape(gt_class_ids)[0] # GT数
miss_match_gt_num = gt_num - tf.shape(
tf.unique(gt_pos_idx)[0])[0] # 未分配anchor的GT
return [
deltas, class_ids, train_rois,
tf_utils.scalar_to_1d_tensor(miss_match_gt_num)
]