def add_generic_rpn_outputs(model, blob_in, dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
是否为目标，对anchor进行回归
"""
loss_gradients = None
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
# 添加proposals层，输出为(bacth_index, x1, y1, x2, y2)
# 1. 获得rpn的输出，MxNxAnchors, MxNx4Anchors;
# 2. 对fpn中的每一层，获取ROIs
FPN.add_fpn_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if cfg.MODEL.FASTER_RCNN:
# CollectAndDistributeFpnRpnProposals also labels proposals when in
# training mode
# 在该函数中构造候选区域的目标值，回归量以及类别，权重
model.CollectAndDistributeFpnRpnProposals()
if model.train:
loss_gradients = FPN.add_fpn_rpn_losses(model)
else:
# Not using FPN, add RPN to a single scale
# 添加输出
add_single_scale_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if model.train:
# 添加loss
loss_gradients = add_single_scale_rpn_losses(model)
return loss_gradients
def distribute_plus_pose(rois, label_blobs, inputs, outputs, train):
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
im_info = inputs[-1].data
# print('inputs[-1].data shape:', im_info.shape)
im_info = im_info[0]
# print('im_info shape:', im_info.shape)
im_scale = im_info[2]
im_h = im_info[0]
im_w = im_info[1]
output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names(train)
blobs = {k: [] for k in output_blob_names}
hg_rois = rois * 1. / im_scale * np.array(
[1, 384.0 / im_w, 384.0 / im_h, 384.0 / im_w, 384.0 / im_h],
dtype=np.float32)
# hg_rois = rois[:, 1:5] * 1. / im_scale * np.array([1, 255.0/im_w, 255.0/im_h, 255.0/im_w, 255.0/im_h], dtype=np.float32)
blobs['rois'] = rois
blobs['rois_hg'] = hg_rois
fpn.add_multilevel_roi_blobs(blobs, 'rois', blobs['rois'], lvls, lvl_min,
lvl_max)
fpn.add_multilevel_roi_blobs(blobs, 'rois_hg', blobs['rois_hg'], lvls,
lvl_min, lvl_max)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn.map_rois_to_fpn_levels(blobs[rois_blob_name][:, 1:5],
lvl_min, lvl_max)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn.add_multilevel_roi_blobs(blobs, rois_blob_name,
blobs[rois_blob_name], target_lvls,
lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn.map_rois_to_fpn_levels(
blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max
)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn.add_multilevel_roi_blobs(
blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min,
lvl_max
)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
# output[0]存储所有的rois
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
# 对fpn的每一层创建新的rois
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
# 选取本层的roi索引
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
# rois中每一项在重新分配到fpn后的连接数组中的位置
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def add_generic_rpn_outputs(model, blob_in, dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
"""
loss_gradients = None
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
FPN.add_fpn_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if cfg.MODEL.FASTER_RCNN:
# CollectAndDistributeFpnRpnProposals also labels proposals when in
# training mode
model.CollectAndDistributeFpnRpnProposals()
if model.train:
loss_gradients = FPN.add_fpn_rpn_losses(model)
return loss_gradients
def add_generic_deep_sup_rpn_outputs(model, blob_in, dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
"""
loss_gradients = None
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
FPN.add_fpn_deep_sup_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if model.train:
if cfg.MODEL.ROI_2CLS_LOSS_OFF:
loss_gradients = None
else:
loss_gradients = FPN.add_fpn_deep_sup_rpn_losses(model)
return loss_gradients
def add_multilevel_pred_box_blob(model, blob_in, pred_boxes_name):
'''
Add pred box blobs for multiple FPN levels to the blobs dict.
parameters:
blob_in: a dict mapping from blob name to numpy ndarray
pred_boxes_name: 'bbox_pred_stage_1' or bbox_pred_stage_2'
'''
workspace.RunNetOnce(model.param_init_net)
lvl_min = cfg.FPN.RPN_MIN_LEVEL
lvl_max = cfg.FPN.RPN_MAX_LEVEL
pred_boxes_name = core.BlobReference(pred_boxes_name)
pred_boxes = workspace.FetchBlob(core.ScopedName(pred_boxes_name))
lvs = fpn.map_rois_to_fpn_levels(pred_boxes, lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(blob_in, pred_boxes_name, pred_boxes, lvs,
lvl_min, lvl_max)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
roi_data.cascade_rcnn.get_cascade_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
###对每一个lvl构建新的输出blob
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
###获取属于该lvl的rois的索引（行数）
idx_lvl = np.where(lvls == lvl)[0]
###将属于该lvl的所有rois放进新的blob中
blob_roi_level = rois[idx_lvl, :]
###将该lvl的blob放入最终的输出blob对应位置中
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
###这里又是在干嘛没看懂
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(blobs, name, blobs[name], lvls, lvl_min,
lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn.map_rois_to_fpn_levels(blobs[rois_blob_name][:, 1:5],
lvl_min, lvl_max)
ws = blobs[rois_blob_name][:, 3] - blobs[rois_blob_name][:, 1] + 1
hs = blobs[rois_blob_name][:, 4] - blobs[rois_blob_name][:, 2] + 1
areas = ws * hs
assert np.all(
areas >= 0
), 'Negative areas founds when add multilevel rois, negative rate: {} / {}, negative anchor: {}, idx of anchor: {}'.format(
np.where(areas < 0)[0].shape, areas.shape[0],
blobs[rois_blob_name][np.where(areas < 0)[0], :],
np.where(areas < 0)[0])
fpn.add_multilevel_roi_blobs(blobs, rois_blob_name,
blobs[rois_blob_name], target_lvls,
lvl_min, lvl_max)
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(
blobs, name, blobs[name], lvls, lvl_min, lvl_max
)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
