def add_generic_rpn_outputs(model, blob_in, dim_in, spatial_scale_in): """Add RPN outputs (objectness classification and bounding box regression) to an RPN model. Abstracts away the use of FPN. 是否为目标,对anchor进行回归 """ loss_gradients = None if cfg.FPN.FPN_ON: # Delegate to the FPN module # 添加proposals层,输出为(bacth_index, x1, y1, x2, y2) # 1. 获得rpn的输出,MxNxAnchors, MxNx4Anchors; # 2. 对fpn中的每一层,获取ROIs FPN.add_fpn_rpn_outputs(model, blob_in, dim_in, spatial_scale_in) if cfg.MODEL.FASTER_RCNN: # CollectAndDistributeFpnRpnProposals also labels proposals when in # training mode # 在该函数中构造候选区域的目标值,回归量以及类别,权重 model.CollectAndDistributeFpnRpnProposals() if model.train: loss_gradients = FPN.add_fpn_rpn_losses(model) else: # Not using FPN, add RPN to a single scale # 添加输出 add_single_scale_rpn_outputs(model, blob_in, dim_in, spatial_scale_in) if model.train: # 添加loss loss_gradients = add_single_scale_rpn_losses(model) return loss_gradients
def distribute_plus_pose(rois, label_blobs, inputs, outputs, train): lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) im_info = inputs[-1].data # print('inputs[-1].data shape:', im_info.shape) im_info = im_info[0] # print('im_info shape:', im_info.shape) im_scale = im_info[2] im_h = im_info[0] im_w = im_info[1] output_blob_names = fast_rcnn_roi_data.get_fast_rcnn_blob_names(train) blobs = {k: [] for k in output_blob_names} hg_rois = rois * 1. / im_scale * np.array( [1, 384.0 / im_w, 384.0 / im_h, 384.0 / im_w, 384.0 / im_h], dtype=np.float32) # hg_rois = rois[:, 1:5] * 1. / im_scale * np.array([1, 255.0/im_w, 255.0/im_h, 255.0/im_w, 255.0/im_h], dtype=np.float32) blobs['rois'] = rois blobs['rois_hg'] = hg_rois fpn.add_multilevel_roi_blobs(blobs, 'rois', blobs['rois'], lvls, lvl_min, lvl_max) fpn.add_multilevel_roi_blobs(blobs, 'rois_hg', blobs['rois_hg'], lvls, lvl_min, lvl_max) for i, k in enumerate(output_blob_names): blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def _distribute_rois_over_fpn_levels(rois_blob_name): """Distribute rois over the different FPN levels.""" # Get target level for each roi # Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take # the box coordinates from columns 1:5 target_lvls = fpn.map_rois_to_fpn_levels(blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max) # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> fpn.add_multilevel_roi_blobs(blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name): """Distribute rois over the different FPN levels.""" # Get target level for each roi # Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take # the box coordinates from columns 1:5 target_lvls = fpn.map_rois_to_fpn_levels( blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max ) # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> fpn.add_multilevel_roi_blobs( blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min, lvl_max )
def distribute(rois, label_blobs, outputs, train): """To understand the output blob order see return value of detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) # output[0]存储所有的rois outputs[0].reshape(rois.shape) outputs[0].data[...] = rois # Create new roi blobs for each FPN level # (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) # 对fpn的每一层创建新的rois rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): # 选取本层的roi索引 idx_lvl = np.where(lvls == lvl)[0] blob_roi_level = rois[idx_lvl, :] outputs[output_idx + 1].reshape(blob_roi_level.shape) outputs[output_idx + 1].data[...] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) # rois中每一项在重新分配到fpn后的连接数组中的位置 rois_idx_restore = np.argsort(rois_idx_order) blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def add_generic_rpn_outputs(model, blob_in, dim_in, spatial_scale_in): """Add RPN outputs (objectness classification and bounding box regression) to an RPN model. Abstracts away the use of FPN. """ loss_gradients = None if cfg.FPN.FPN_ON: # Delegate to the FPN module FPN.add_fpn_rpn_outputs(model, blob_in, dim_in, spatial_scale_in) if cfg.MODEL.FASTER_RCNN: # CollectAndDistributeFpnRpnProposals also labels proposals when in # training mode model.CollectAndDistributeFpnRpnProposals() if model.train: loss_gradients = FPN.add_fpn_rpn_losses(model) return loss_gradients
def add_generic_deep_sup_rpn_outputs(model, blob_in, dim_in, spatial_scale_in): """Add RPN outputs (objectness classification and bounding box regression) to an RPN model. Abstracts away the use of FPN. """ loss_gradients = None if cfg.FPN.FPN_ON: # Delegate to the FPN module FPN.add_fpn_deep_sup_rpn_outputs(model, blob_in, dim_in, spatial_scale_in) if model.train: if cfg.MODEL.ROI_2CLS_LOSS_OFF: loss_gradients = None else: loss_gradients = FPN.add_fpn_deep_sup_rpn_losses(model) return loss_gradients
def add_multilevel_pred_box_blob(model, blob_in, pred_boxes_name): ''' Add pred box blobs for multiple FPN levels to the blobs dict. parameters: blob_in: a dict mapping from blob name to numpy ndarray pred_boxes_name: 'bbox_pred_stage_1' or bbox_pred_stage_2' ''' workspace.RunNetOnce(model.param_init_net) lvl_min = cfg.FPN.RPN_MIN_LEVEL lvl_max = cfg.FPN.RPN_MAX_LEVEL pred_boxes_name = core.BlobReference(pred_boxes_name) pred_boxes = workspace.FetchBlob(core.ScopedName(pred_boxes_name)) lvs = fpn.map_rois_to_fpn_levels(pred_boxes, lvl_min, lvl_max) fpn.add_multilevel_roi_blobs(blob_in, pred_boxes_name, pred_boxes, lvs, lvl_min, lvl_max)
def distribute(rois, label_blobs, outputs, train): """To understand the output blob order see return value of roi_data.cascade_rcnn.get_cascade_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) outputs[0].reshape(rois.shape) outputs[0].data[...] = rois # Create new roi blobs for each FPN level ###对每一个lvl构建新的输出blob # (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): ###获取属于该lvl的rois的索引(行数) idx_lvl = np.where(lvls == lvl)[0] ###将属于该lvl的所有rois放进新的blob中 blob_roi_level = rois[idx_lvl, :] ###将该lvl的blob放入最终的输出blob对应位置中 outputs[output_idx + 1].reshape(blob_roi_level.shape) outputs[output_idx + 1].data[...] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) ###这里又是在干嘛没看懂 rois_idx_restore = np.argsort(rois_idx_order) blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def _add_multilevel_rois_for_test(blobs, name): """Distributes a set of RoIs across FPN pyramid levels by creating new level specific RoI blobs. Arguments: blobs (dict): dictionary of blobs name (str): a key in 'blobs' identifying the source RoI blob Returns: [by ref] blobs (dict): new keys named by `name + 'fpn' + level` are added to dict each with a value that's an R_level x 5 ndarray of RoIs (see _get_rois_blob for format) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max) fpn.add_multilevel_roi_blobs(blobs, name, blobs[name], lvls, lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name): """Distribute rois over the different FPN levels.""" # Get target level for each roi # Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take # the box coordinates from columns 1:5 target_lvls = fpn.map_rois_to_fpn_levels(blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max) ws = blobs[rois_blob_name][:, 3] - blobs[rois_blob_name][:, 1] + 1 hs = blobs[rois_blob_name][:, 4] - blobs[rois_blob_name][:, 2] + 1 areas = ws * hs assert np.all( areas >= 0 ), 'Negative areas founds when add multilevel rois, negative rate: {} / {}, negative anchor: {}, idx of anchor: {}'.format( np.where(areas < 0)[0].shape, areas.shape[0], blobs[rois_blob_name][np.where(areas < 0)[0], :], np.where(areas < 0)[0]) fpn.add_multilevel_roi_blobs(blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min, lvl_max)
def _add_multilevel_rois_for_test(blobs, name): """Distributes a set of RoIs across FPN pyramid levels by creating new level specific RoI blobs. Arguments: blobs (dict): dictionary of blobs name (str): a key in 'blobs' identifying the source RoI blob Returns: [by ref] blobs (dict): new keys named by `name + 'fpn' + level` are added to dict each with a value that's an R_level x 5 ndarray of RoIs (see _get_rois_blob for format) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max) fpn.add_multilevel_roi_blobs( blobs, name, blobs[name], lvls, lvl_min, lvl_max )
def distribute(rois, label_blobs, outputs, train): """To understand the output blob order see return value of detectron.roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) outputs[0].reshape(rois.shape) outputs[0].data[...] = rois # Create new roi blobs for each FPN level # (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): idx_lvl = np.where(lvls == lvl)[0] blob_roi_level = rois[idx_lvl, :] outputs[output_idx + 1].reshape(blob_roi_level.shape) outputs[output_idx + 1].data[...] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) rois_idx_restore = np.argsort(rois_idx_order) blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])