def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn.map_rois_to_fpn_levels(blobs[rois_blob_name][:, 1:5],
lvl_min, lvl_max)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn.add_multilevel_roi_blobs(blobs, rois_blob_name,
blobs[rois_blob_name], target_lvls,
lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn.map_rois_to_fpn_levels(
blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max
)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn.add_multilevel_roi_blobs(
blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min,
lvl_max
)
def _add_multilevel_rois(blobs):
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# The map_rois_to_fpn_levels and add_multilevel_roi_blobs functions are
# shared among the 2D and 3D models.
lvls = fpn.map_rois_to_fpn_levels(blobs['rois'][:, 1:], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(blobs, 'rois', blobs['rois'], lvls, lvl_min,
lvl_max)
if cfg.MODEL.MASK_ON:
# Masks use the same rois as the box/cls head
fpn.add_multilevel_roi_blobs(blobs,
'mask_rois',
blobs['rois'],
lvls,
lvl_min,
lvl_max,
valid_levels=blobs['roi_has_mask_int32'])
if cfg.MODEL.KEYPOINTS_ON:
# Keypoints use a separate set of training rois
lvls = fpn.map_rois_to_fpn_levels(blobs['keypoint_rois'][:, 1:],
lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(blobs, 'keypoint_rois',
blobs['keypoint_rois'], lvls, lvl_min,
lvl_max)
def distribute(self, rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
# Put all rois into rois without distribute
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Distribute rois into different level according to map method
if not cfg.PAN.AdaptivePooling_ON:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
# Create new roi blobs for each FPN level
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32),
outputs[-1])
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(blobs[name][:, 1:], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(
blobs, name, blobs[name], lvls, lvl_min, lvl_max)
def _add_multilevel_rois(blobs):
"""By default training RoIs are added for a single feature map level only.
When using FPN, the RoIs must be distributed over different FPN levels
according the level assignment heuristic (see: modeling.FPN.
map_rois_to_fpn_levels).
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_name = 'mask_rois'
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn.map_rois_to_fpn_levels(blobs[rois_blob_name][:, 1:5],
lvl_min, lvl_max)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn.add_multilevel_roi_blobs(blobs, rois_blob_name, blobs[rois_blob_name],
target_lvls, lvl_min, lvl_max)
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(blobs, name, blobs[name], lvls, lvl_min,
lvl_max)
def generic_rpn_outputs(dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
"""
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
return FPN.fpn_rpn_outputs(dim_in, spatial_scale_in)
else:
# Not using FPN, add RPN to a single scale
return single_scale_rpn_outputs(dim_in, spatial_scale_in)
def generic_rpn_outputs_panet(dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
"""
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
return FPN.fpn_rpn_outputs(dim_in, spatial_scale_in)
else:
# Not using FPN, add RPN to a single scale
return single_scale_rpn_outputs(dim_in, spatial_scale_in)
def add_generic_rpn_outputs(model, blob_in, dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
"""
loss_gradients = None
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
FPN.add_fpn_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if cfg.MODEL.FASTER_RCNN:
# CollectAndDistributeFpnRpnProposals also labels proposals when in
# training mode
model.CollectAndDistributeFpnRpnProposals()
if model.train:
loss_gradients = FPN.add_fpn_rpn_losses(model)
else:
# Not using FPN, add RPN to a single scale
add_single_scale_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if model.train:
loss_gradients = add_single_scale_rpn_losses(model)
return loss_gradients
def add_generic_rpn_outputs(model, blob_in, dim_in, spatial_scale_in):
"""Add RPN outputs (objectness classification and bounding box regression)
to an RPN model. Abstracts away the use of FPN.
"""
loss_gradients = None
if cfg.FPN.FPN_ON:
# Delegate to the FPN module
FPN.add_fpn_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if cfg.MODEL.FASTER_RCNN:
# CollectAndDistributeFpnRpnProposals also labels proposals when in
# training mode
model.CollectAndDistributeFpnRpnProposals()
if model.train:
loss_gradients = FPN.add_fpn_rpn_losses(model)
else:
# Not using FPN, add RPN to a single scale
add_single_scale_rpn_outputs(model, blob_in, dim_in, spatial_scale_in)
if model.train:
loss_gradients = add_single_scale_rpn_losses(model)
return loss_gradients
def _add_multilevel_rois(blobs):
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# The map_rois_to_fpn_levels and add_multilevel_roi_blobs functions are
# shared among the 2D and 3D models.
lvls = fpn.map_rois_to_fpn_levels(blobs['rois'][:, 1:], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(
blobs, 'rois', blobs['rois'], lvls, lvl_min, lvl_max)
if cfg.MODEL.MASK_ON:
# Masks use the same rois as the box/cls head
fpn.add_multilevel_roi_blobs(
blobs, 'mask_rois', blobs['rois'], lvls, lvl_min, lvl_max,
valid_levels=blobs['roi_has_mask_int32'])
if cfg.MODEL.KEYPOINTS_ON:
# Keypoints use a separate set of training rois
lvls = fpn.map_rois_to_fpn_levels(
blobs['keypoint_rois'][:, 1:], lvl_min, lvl_max)
fpn.add_multilevel_roi_blobs(
blobs, 'keypoint_rois', blobs['keypoint_rois'], lvls, lvl_min,
lvl_max)
def distribute(rois, label_blobs, outputs, train):
"""To understand the output blob order see return value of
roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
outputs[0].reshape(rois.shape)
outputs[0].data[...] = rois
# Create new roi blobs for each FPN level
# (See: modeling.FPN.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1].reshape(blob_roi_level.shape)
outputs[output_idx + 1].data[...] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
rois_idx_restore = np.argsort(rois_idx_order)
blob_utils.py_op_copy_blob(rois_idx_restore.astype(np.int32), outputs[-1])
def _single_gpu_build_func(model):
"""Builds the model on a single GPU. Can be called in a loop over GPUs
with name and device scoping to create a data parallel model."""
# Some generic tensors
model.ConstantFill([], 'zero', shape=[1], value=0)
model.ConstantFill([], 'minus1', shape=[1], value=-1)
blob, dim, spatial_scale = add_conv_body_func(model)
if freeze_conv_body:
model.StopGradient(blob, blob)
if cfg.MODEL.VIDEO_ON:
blob = time_pool_blobs(
blob, model, cfg.VIDEO.BODY_HEAD_LINK)
if not model.train:
# Create a net that can be used to compute the conv body only on an
# image (no RPN or heads / branches)
model.conv_body_net = model.net.Clone('conv_body_net')
if cfg.MODEL.VIDEO_ON and cfg.VIDEO.BODY_HEAD_LINK == '':
FPN_lib = FPN3D
head_3d = True
out_time_dim = cfg.VIDEO.NUM_FRAMES_MID
else:
FPN_lib = FPN
head_3d = False
out_time_dim = 1
if cfg.FPN.FPN_ON:
FPN_lib.add_fpn_rpn_outputs(model, blob, dim, spatial_scale,
time_dim=out_time_dim)
if model.train:
loss_gradients = FPN.add_fpn_rpn_losses(model)
else:
add_rpn_outputs(model, blob, dim, spatial_scale, nd=head_3d,
time_dim=out_time_dim)
if model.train:
loss_gradients = add_rpn_losses(model,
time_dim=out_time_dim)
return loss_gradients if model.train else None
def generic_rpn_losses(*inputs, **kwargs):
"""Add RPN losses. Abstracts away the use of FPN."""
if cfg.FPN.FPN_ON:
return FPN.fpn_rpn_losses(*inputs, **kwargs)
else:
return single_scale_rpn_losses(*inputs, **kwargs)
def generic_rpn_losses(*inputs, **kwargs):
"""Add RPN losses. Abstracts away the use of FPN."""
if cfg.FPN.FPN_ON:
return FPN.fpn_rpn_losses(*inputs, **kwargs)
else:
return single_scale_rpn_losses(*inputs, **kwargs)
def _single_gpu_build_func(model):
"""Builds the model on a single GPU. Can be called in a loop over GPUs
with name and device scoping to create a data parallel model."""
# For training we define one net that contains all ops
# For inference, we split the graph into two nets: a standard fast r-cnn
# net and a mask prediction net; the mask net is only applied to a
# subset of high-scoring detections
is_inference = not model.train
# Some generic tensors
model.ConstantFill([], 'zero', shape=[1], value=0)
model.ConstantFill([], 'minus1', shape=[1], value=-1)
# Add the conv body
blob_conv, dim_conv, spatial_scale_conv = add_conv_body_func(model)
if freeze_conv_body:
for b in blob_ref_to_list(blob_conv):
model.StopGradient(b, b)
# Convert from 3D blob to 2D, in case of videos to attach a 2D head
# (not necessarily will happen though)
if cfg.MODEL.VIDEO_ON:
blob_conv = time_pool_blobs(
blob_conv, model, cfg.VIDEO.BODY_HEAD_LINK)
if is_inference:
# Create a net that can be used to compute the conv body only on an
# image (no RPN or heads / branches)
model.conv_body_net = model.net.Clone('conv_body_net')
# Select the FPN lib, based on whether the head is 3D or 2D
if cfg.MODEL.VIDEO_ON and cfg.VIDEO.BODY_HEAD_LINK == '':
FPN_lib = FPN3D
head_3d = True
out_time_dim = cfg.VIDEO.NUM_FRAMES_MID
else:
FPN_lib = FPN
head_3d = False
out_time_dim = 1
# Add the RPN branch
if cfg.MODEL.FASTER_RCNN:
if cfg.FPN.FPN_ON:
FPN_lib.add_fpn_rpn_outputs(
model, blob_conv, dim_conv, spatial_scale_conv,
time_dim=out_time_dim)
model.CollectAndDistributeFpnRpnProposals()
else:
add_rpn_outputs(model, blob_conv, dim_conv, spatial_scale_conv,
nd=head_3d, time_dim=out_time_dim)
if cfg.FPN.FPN_ON:
# Code only supports case when RPN and ROI min levels are the same
assert cfg.FPN.RPN_MIN_LEVEL == cfg.FPN.ROI_MIN_LEVEL
# FPN RPN max level might be > FPN ROI max level in which case we
# need to discard some leading conv blobs (blobs are ordered from
# max level to min level)
num_roi_levels = cfg.FPN.ROI_MAX_LEVEL - cfg.FPN.ROI_MIN_LEVEL + 1
blob_conv = blob_conv[-num_roi_levels:]
spatial_scale_conv = spatial_scale_conv[-num_roi_levels:]
# Add the Fast R-CNN branch
blob_frcn, dim_frcn, spatial_scale_frcn = add_roi_frcn_head_func(
model, blob_conv, dim_conv, spatial_scale_conv)
add_fast_rcnn_outputs(model, blob_frcn, dim_frcn, is_head_3d=head_3d)
# Add the mask branch
if cfg.MODEL.MASK_ON:
if is_inference:
bbox_net = copy.deepcopy(model.net.Proto())
# Add the mask branch
blob_mrcn, dim_mrcn, _ = add_roi_mask_head_func(
model, blob_conv, dim_conv, spatial_scale_conv)
blob_mask = add_mask_rcnn_outputs(model, blob_mrcn, dim_mrcn)
if is_inference:
# Extract the mask prediction net, store it as its own network,
# then restore the primary net to the bbox-only network
model.mask_net, blob_mask = get_suffix_net(
'mask_net', bbox_net.op, model.net, [blob_mask])
model.net._net = bbox_net
# Add the keypoint branch
if cfg.MODEL.KEYPOINTS_ON:
if is_inference:
bbox_net = copy.deepcopy(model.net.Proto())
blob_krcnn, dim_krcnn, _ = add_roi_keypoint_head_func(
model, blob_conv, dim_conv, spatial_scale_conv)
blob_keypoint = add_heatmap_outputs(
model, blob_krcnn, dim_krcnn,
time_dim=out_time_dim, is_head_3d=head_3d)
if is_inference:
model.keypoint_net, keypoint_blob_out = get_suffix_net(
'keypoint_net', bbox_net.op, model.net, [blob_keypoint])
model.net._net = bbox_net
if model.train:
loss_gradients = add_fast_rcnn_losses(model, time_dim=out_time_dim)
if cfg.MODEL.MASK_ON:
loss_gradients.update(add_mask_rcnn_losses(model, blob_mask,
time_dim=out_time_dim))
if cfg.MODEL.KEYPOINTS_ON:
loss_gradients.update(add_heatmap_losses(model, time_dim=out_time_dim))
if cfg.MODEL.FASTER_RCNN:
if cfg.FPN.FPN_ON:
# The loss function is shared between 2D and 3D FPN
loss_gradients.update(FPN.add_fpn_rpn_losses(
model, time_dim=out_time_dim))
if cfg.VIDEO.PREDICT_RPN_BOX_VIS:
loss_gradients.update(FPN.add_fpn_rpn_vis_losses(
model, time_dim=out_time_dim))
else:
loss_gradients.update(add_rpn_losses(
model, time_dim=out_time_dim))
if cfg.VIDEO.PREDICT_RPN_BOX_VIS:
loss_gradients.update(add_rpn_vis_losses(
model, time_dim=out_time_dim))
return loss_gradients if model.train else None
