def _elementwise_linear(
model, op_call, blob_in, blob_out, dim,
weight_init=None, bias_init=None, **kwargs
):
"""Elementwise_Linear"""
weight_init = weight_init or ('ConstantFill', {'value': 1.0})
bias_init = bias_init or ('ConstantFill', {'value': 0.0})
blob_out = blob_out or model.net.NextName()
if model.init_params:
weight = model.param_init_net.__getattr__(weight_init[0])(
[],
blob_out + '_w',
shape=[dim],
**weight_init[1]
)
bias = model.param_init_net.__getattr__(bias_init[0])(
[],
blob_out + '_b',
shape=[dim],
**bias_init[1]
)
else:
weight = core.ScopedBlobReference(
blob_out + '_w', model.param_init_net)
bias = core.ScopedBlobReference(
blob_out + '_b', model.param_init_net)
model.AddParameter(weight, ParameterTags.WEIGHT)
model.AddParameter(bias, ParameterTags.BIAS)
return op_call([blob_in, weight, bias], blob_out, **kwargs)
def _FC_or_packed_FC(
self, op_call, blob_in, blob_out, dim_in, dim_out, weight_init=None,
bias_init=None, **kwargs
):
"""FC"""
weight_init = weight_init if weight_init else ('XavierFill', {})
bias_init = bias_init if bias_init else ('ConstantFill', {})
blob_out = blob_out or self.net.NextName()
if self.init_params:
weight = self.param_init_net.__getattr__(weight_init[0])(
[],
blob_out + '_w',
shape=[dim_out, dim_in],
**weight_init[1]
)
bias = self.param_init_net.__getattr__(bias_init[0])(
[],
blob_out + '_b',
shape=[dim_out, ],
**bias_init[1]
)
else:
weight = core.ScopedBlobReference(
blob_out + '_w', self.param_init_net)
bias = core.ScopedBlobReference(
blob_out + '_b', self.param_init_net)
if 'freeze_bias' in kwargs:
self.params.extend([weight])
else:
self.params.extend([weight, bias])
self.weights.append(weight)
self.biases.append(bias)
return op_call([blob_in, weight, bias], blob_out, **kwargs)
def CollectAndDistributeFpnRpnProposals(self):
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
# Prepare input blobs
rois_names = ['rpn_rois_fpn' + str(l) for l in range(k_min, k_max + 1)]
score_names = [
'rpn_roi_probs_fpn' + str(l) for l in range(k_min, k_max + 1)
]
blobs_in = rois_names + score_names
if self.train:
blobs_in += ['roidb', 'im_info']
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'CollectAndDistributeFpnRpnProposalsOp:' + ','.join(
[str(b) for b in blobs_in])
# Prepare output blobs
blobs_out = fast_rcnn_roi_data.get_fast_rcnn_blob_names(
is_training=self.train)
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
outputs = self.net.Python(
CollectAndDistributeFpnRpnProposalsOp(self.train).forward)(
blobs_in, blobs_out, name=name)
return outputs
def GenerateKeypointIndicators(
self,
blobs_in,
blob_out,
blob_rois='keypoint_rois',
):
""" Add keypoint indicators to the refine network. It maps the
keypoint heatmap on local rois into the expanded rois. The heatmap
is re-drawed instead of directly resized to handle special issues
for keypoints
Input blobs: [data, kps_scores]
Input rois: keypoint_rois
Output blob: keypoint_indicators
"""
blob_rois = core.ScopedBlobReference(blob_rois) # refer blob_rois
blobs_in_list = blobs_in + [blob_rois]
name = 'GenerateKeypointIndicatorsOp:' + ','.join(
[str(b) for b in blobs_in_list]
)
blob_out = core.ScopedBlobReference(blob_out)
grad_input_indices = [0] # ignore gradient for blob_rois
up_scale = cfg.REFINENET.UP_SCALE
M = cfg.REFINENET.ROI_XFORM_RESOLUTION
xform_out = self.net.Python(
GenerateKeypointIndicatorsOp(up_scale=up_scale, resolution=M).forward,
GenerateKeypointIndicatorsOp(up_scale=up_scale, resolution=M).backward,
grad_input_indices=grad_input_indices
)(blobs_in_list, blob_out, name=name)
return xform_out
def AffineChannelNd(self,
blob_in,
blob_out,
dim_out,
share_with=None,
inplace=False):
if cfg.MODEL.USE_BN:
return self.SpatialBNLayer(blob_in, blob_out, dim_out, share_with,
inplace)
blob_out = blob_out or self.net.NextName()
is_not_sharing = share_with is None
param_prefix = blob_out if is_not_sharing else share_with
scale = core.ScopedBlobReference(param_prefix + '_s',
self.param_init_net)
bias = core.ScopedBlobReference(param_prefix + '_b',
self.param_init_net)
if is_not_sharing:
self.net.Proto().external_input.extend([str(scale), str(bias)])
self.params.extend([scale, bias])
self.weights.append(scale)
self.biases.append(bias)
if inplace:
return self.net.AffineChannelNd([blob_in, scale, bias], blob_in)
else:
return self.net.AffineChannelNd([blob_in, scale, bias], blob_out)
def Adagrad(self, net, param_init_net,
param, grad, alpha, epsilon, sparse_dedup_aggregator=None,
engine=''):
if alpha <= 0:
return
param_square_sum = param_init_net.ConstantFill(
[param],
core.ScopedBlobReference(param + "_square_sum"),
value=0.0
)
# Set learning rate to negative so that we can add the grad to param
# directly later.
lr = param_init_net.ConstantFill(
[], core.ScopedBlobReference(param + "_lr"), value=-alpha)
if isinstance(grad, core.GradientSlice):
if sparse_dedup_aggregator:
grad = net.DeduplicateGradientSlices(
grad, aggregator=sparse_dedup_aggregator)
net.SparseAdagrad(
[param, param_square_sum, grad.indices, grad.values, lr],
[param, param_square_sum],
epsilon=epsilon,
engine=engine
)
else:
net.Adagrad(
[param, param_square_sum, grad, lr],
[param, param_square_sum],
epsilon=epsilon,
engine=engine
)
def FC(self,
blob_in,
blob_out,
dim_in,
dim_out,
weight_init=None,
bias_init=None,
**kwargs):
"""FC"""
weight_init = weight_init if weight_init else ('XavierFill', {})
bias_init = bias_init if bias_init else ('ConstantFill', {})
blob_out = blob_out or self.net.NextName()
if self.init_params:
weight = self.param_init_net.__getattr__(
weight_init[0])([],
blob_out + '_w',
shape=[dim_out, dim_in],
**weight_init[1])
bias = self.param_init_net.__getattr__(
bias_init[0])([],
blob_out + '_b',
shape=[
dim_out,
],
**bias_init[1])
else:
weight = core.ScopedBlobReference(blob_out + '_w',
self.param_init_net)
bias = core.ScopedBlobReference(blob_out + '_b',
self.param_init_net)
self.params.extend([weight, bias])
return self.net.FC([blob_in, weight, bias], blob_out, **kwargs)
def PrepareLabelsForPRNAndUpdateRefineBlobs(self):
""" Prepare labels for PRN and update blobs for RefineNet.
Input blobs: ['mask_ious', 'labels_int32']
- labels_int32 is the cls label for rois
If used during training, adds the related blobs for the specific
refinement task, such as ['refined_masks_int32'].
Output blob: ['prn_labels_int32', 'roi_needs_refine_int32']
- prn_labels_int32 is the labels for prn.
- roi_needs_refine_int32 is a binary label indicates whether
further refinement is needed or not.
And if used during training, also doing inplace-update for the
labels of refinement tasks. Such as update ['refined_masks_int32']
"""
# Prepare input blobs
blobs_in = prn_label_op.get_op_blob_in_names()
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'PrepareLabelsForPRNAndUpdateRefineBlobsOp: ' + ','.join([str(b) for b in blobs_in])
# Prepare output blobs
blobs_out = prn_label_op.get_op_blob_out_names()
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
# Execute op
output = self.net.Python(
PrepareLabelsForPRNAndUpdateRefineBlobsOp().forward
)(blobs_in, blobs_out, name=name)
return output
def RescaleAndDumplicateFeatureSingle(
self,
blobs_in,
blob_out,
blob_rois,
src_spatial_scales,
dst_spatial_scale
):
""" Dumplicate feature maps for the refiner network.
If use FPN, then rescale the different FPN level feature
to a dst_spatial_scale. Then concancate the feature maps
along the batch dimension
Input blobs: res_...
Input rois: mask_rois_fpn
Output blobs: rois_global_feature
"""
# Single scale feature
src_sc = src_spatial_scales
dst_sc = dst_spatial_scale
blobs_in_list = [blobs_in, core.ScopedBlobReference(blob_rois)]
name = 'RescaleAndDumplicateOp:' + ','.join(
[str(b) for b in blobs_in_list]
)
blob_out = core.ScopedBlobReference(blob_out)
xform_out = self.net.Python(
RescaleAndDumplicateFeatureSingleOp(src_sc, dst_sc).forward,
RescaleAndDumplicateFeatureSingleOp(src_sc, dst_sc).backward,
grad_input_indices=[0]
)(blobs_in_list, blob_out, name=name)
return xform_out
def PoolingIndicatorFeatureSingle(
self,
blobs_in,
blob_out,
blob_rois,
spatial_scale
):
""" Pool indicator feature for the rois. Scale the roi with a
factor and then create a feature map with size MxM.
Input blobs: res_...
Input rois: mask_rois_fpn
Output blobs: rois_global_feature
"""
M = cfg.REFINENET.RESOLUTION
up_scale = cfg.REFINENET.UP_SCALE
blobs_in_list = [blobs_in, core.ScopedBlobReference(blob_rois)]
name = 'PoolingIndicatorFeatureSingleOp:' + ','.join(
[str(b) for b in blobs_in_list]
)
blob_out = core.ScopedBlobReference(blob_out)
xform_out = self.net.Python(
PoolingIndicatorFeatureSingleOp(spatial_scale, up_scale, M).forward,
PoolingIndicatorFeatureSingleOp(spatial_scale, up_scale, M).backward,
grad_input_indices=[0]
)(blobs_in_list, blob_out, name=name)
return xform_out
def GenerateLocalMaskIndicators(
self,
blobs_in,
blob_out,
blob_rois='mask_rois',
):
""" Add mask indicators to the refine network. It maps the
'mask_probs' into the input images' space, and narrow it down
by the value 'scale'
Input blobs: [data, mask_probs]
Input rois: mask_rois
Output blob: mask_indicators
"""
blob_rois = core.ScopedBlobReference(blob_rois) # refer blob_rois
blobs_in_list = blobs_in + [blob_rois]
name = 'GenerateMaskIndicatorsOp:' + ','.join(
[str(b) for b in blobs_in_list]
)
blob_out = core.ScopedBlobReference(blob_out)
grad_input_indices=[0] # ignore gradient for blob_rois
up_scale = cfg.REFINENET.UP_SCALE
M = cfg.REFINENET.ROI_XFORM_RESOLUTION
xform_out = self.net.Python(
GenerateLocalMaskIndicatorsOp(up_scale=up_scale, resolution=M).forward,
GenerateLocalMaskIndicatorsOp(up_scale=up_scale, resolution=M).backward,
grad_input_indices=grad_input_indices
)(blobs_in_list, blob_out, name=name)
return xform_out
def GenerateGlobalMaskIndicators(
self,
blobs_in,
blob_out,
blob_rois='mask_rois',
dst_spatial_scale=1/16.
):
""" Add mask indicators to the refine network. It maps the
'mask_probs' into the input images' space, and narrow it down
by the value 'scale'
Input blobs: [data, mask_probs]
Input rois: mask_rois
Output blob: mask_indicators
"""
blob_rois = core.ScopedBlobReference(blob_rois) # refer blob_rois
blobs_in_list = blobs_in + [blob_rois]
name = 'GenerateMaskIndicatorsOp:' + ','.join(
[str(b) for b in blobs_in_list]
)
blob_out = core.ScopedBlobReference(blob_out)
grad_input_indices=[0] # ignore gradient for blob_rois
xform_out = self.net.Python(
GenerateGlobalMaskIndicatorsOp(scale=dst_spatial_scale).forward,
GenerateGlobalMaskIndicatorsOp(scale=dst_spatial_scale).backward,
grad_input_indices=grad_input_indices
)(blobs_in_list, blob_out, name=name)
return xform_out
def GroupConv(self,
blob_in,
blob_out,
dim_in,
dim_out,
kernel,
weight_init,
bias_init,
group=1,
**kwargs):
"""Convolution. We intentionally do not provide odd kernel/stride/pad
settings in order to discourage the use of odd cases.
"""
if self.use_cudnn:
kwargs['engine'] = 'CUDNN'
kwargs['exhaustive_search'] = self.cudnn_exhaustive_search
if self.ws_nbytes_limit:
kwargs['ws_nbytes_limit'] = self.ws_nbytes_limit
if dim_in % group:
raise ValueError("dim_in should be divisible by group.")
splitted_blobs = self.net.DepthSplit(
blob_in,
['_' + blob_out + '_gconv_split_' + str(i) for i in range(group)],
dimensions=[dim_in / group for i in range(group)],
order=self.order)
weight_shape = ([dim_out / group, dim_in /
group, kernel, kernel] if self.order == "NCHW" else
[dim_out / group, kernel, kernel, dim_in / group])
conv_blobs = []
for i in range(group):
if self.init_params:
weight = self.param_init_net.__getattr__(
weight_init[0])([],
blob_out + '_gconv_%d_w' % i,
shape=weight_shape,
**weight_init[1])
bias = self.param_init_net.__getattr__(
bias_init[0])([],
blob_out + '_gconv_%d_b' % i,
shape=[dim_out / group],
**bias_init[1])
else:
weight = core.ScopedBlobReference(blob_out + '_gconv_%d_w' % i,
self.param_init_net)
bias = core.ScopedBlobReference(blob_out + '_gconv_%d_b' % i,
self.param_init_net)
self.params.extend([weight, bias])
self.weights.append(weight)
self.biases.append(bias)
conv_blobs.append(splitted_blobs[i].Conv([weight, bias],
blob_out +
'_gconv_%d' % i,
kernel=kernel,
order=self.order,
**kwargs))
concat, concat_dims = self.net.Concat(
conv_blobs, [blob_out, "_" + blob_out + "_concat_dims"],
order=self.order)
return concat
def spatial_bn(model,
blob_in,
blob_out,
dim_in,
init_scale=1.,
init_bias=0.,
order="NCHW",
**kwargs):
blob_out = blob_out or model.net.NextName()
# Input: input, scale, bias, est_mean, est_inv_var
# Output: output, running_mean, running_inv_var, saved_mean,
# saved_inv_var
# scale: initialize with init_scale (default 1.)
# bias: initialize with init_bias (default 0.)
# est mean: zero
# est var: ones
def init_blob(value, suffix):
return model.param_init_net.ConstantFill([],
blob_out + "_" + suffix,
shape=[dim_in],
value=value)
if model.init_params:
scale, bias = init_blob(init_scale, "s"), init_blob(init_bias, "b")
running_mean = init_blob(0.0, "rm")
running_inv_var = init_blob(1.0, "riv")
else:
scale = core.ScopedBlobReference(blob_out + '_s', model.param_init_net)
bias = core.ScopedBlobReference(blob_out + '_b', model.param_init_net)
running_mean = core.ScopedBlobReference(blob_out + '_rm',
model.param_init_net)
running_inv_var = core.ScopedBlobReference(blob_out + '_riv',
model.param_init_net)
model.AddParameter(running_mean, ParameterTags.COMPUTED_PARAM)
model.AddParameter(running_inv_var, ParameterTags.COMPUTED_PARAM)
model.AddParameter(scale, ParameterTags.WEIGHT)
model.AddParameter(bias, ParameterTags.BIAS)
blob_outs = [
blob_out, running_mean, running_inv_var, blob_out + "_sm",
blob_out + "_siv"
]
if 'is_test' in kwargs and kwargs['is_test']:
blob_outputs = model.net.SpatialBN(
[blob_in, scale, bias, blob_outs[1], blob_outs[2]], [blob_out],
order=order,
**kwargs)
return blob_outputs
else:
blob_outputs = model.net.SpatialBN(
[blob_in, scale, bias, blob_outs[1], blob_outs[2]],
blob_outs,
order=order,
**kwargs)
# Return the output
return blob_outputs[0]
def CollectAndDistributeFpnClusterProposals(self):
"""Merge RPN proposals generated at multiple FPN levels and then
distribute those proposals to their appropriate FPN levels. An anchor
at one FPN level may predict an RoI that will map to another level,
hence the need to redistribute the proposals.
This function assumes standard blob names for input and output blobs.
Input blobs: [rpn_rois_fpn<min>, ..., rpn_rois_fpn<max>,
rpn_roi_probs_fpn<min>, ..., rpn_roi_probs_fpn<max>]
- rpn_rois_fpn<i> are the RPN proposals for FPN level i; see rpn_rois
documentation from GenerateProposals.
- rpn_roi_probs_fpn<i> are the RPN objectness probabilities for FPN
level i; see rpn_roi_probs documentation from GenerateProposals.
If used during training, then the input blobs will also include:
[roidb, im_info] (see GenerateProposalLabels).
Output blobs: [rois_fpn<min>, ..., rois_rpn<max>, rois,
rois_idx_restore]
- rois_fpn<i> are the RPN proposals for FPN level i
- rois_idx_restore is a permutation on the concatenation of all
rois_fpn<i>, i=min...max, such that when applied the RPN RoIs are
restored to their original order in the input blobs.
If used during training, then the output blobs will also include:
[labels, bbox_targets, bbox_inside_weights, bbox_outside_weights].
"""
# Prepare input blobs
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
# Prepare input blobs
rois_names = ['rpn_rois_fpn' + str(l) for l in range(k_min, k_max + 1)]
score_names = [
'rpn_roi_probs_fpn' + str(l) for l in range(k_min, k_max + 1)
]
blobs_in = rois_names + score_names
if self.train:
blobs_in += ['roidb', 'im_info']
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'CollectAndDistributeFpnClusterProposalsOp:' + ','.join(
[str(b) for b in blobs_in]
)
# Prepare output blobs
blobs_out = cluster_rcnn_roi_data.get_cluster_rcnn_blob_names(
is_training=self.train
)
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
outputs = self.net.Python(
CollectAndDistributeFpnClusterProposalsOp(self.train).forward
)(blobs_in, blobs_out, name=name)
return outputs
def conv_transpose(
model,
blob_in,
blob_out,
dim_in,
dim_out,
kernel,
weight_init=None,
bias_init=None,
use_cudnn=False,
order="NCHW",
cudnn_exhaustive_search=False,
ws_nbytes_limit=None,
**kwargs
):
"""ConvTranspose.
"""
weight_init = weight_init if weight_init else ('XavierFill', {})
bias_init = bias_init if bias_init else ('ConstantFill', {})
blob_out = blob_out or model.net.NextName()
weight_shape = (
[dim_in, dim_out, kernel, kernel]
if order == "NCHW" else [dim_in, kernel, kernel, dim_out]
)
if model.init_params:
weight = model.param_init_net.__getattr__(weight_init[0])(
[],
blob_out + '_w',
shape=weight_shape,
**weight_init[1]
)
bias = model.param_init_net.__getattr__(bias_init[0])(
[],
blob_out + '_b',
shape=[dim_out, ],
**bias_init[1]
)
else:
weight = core.ScopedBlobReference(
blob_out + '_w', model.param_init_net)
bias = core.ScopedBlobReference(
blob_out + '_b', model.param_init_net)
model.AddParameter(weight, ParameterTags.WEIGHT)
model.AddParameter(bias, ParameterTags.BIAS)
if use_cudnn:
kwargs['engine'] = 'CUDNN'
kwargs['exhaustive_search'] = cudnn_exhaustive_search
if ws_nbytes_limit:
kwargs['ws_nbytes_limit'] = ws_nbytes_limit
return model.net.ConvTranspose(
[blob_in, weight, bias],
blob_out,
kernel=kernel,
order=order,
**kwargs
)
def mySplit(self):
blobs_in = ['fc7', 'roi_numbers']
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
blobs_out = ['fc7_oldC', 'fc7_newC']
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
self.net.Python(mySplitOp(self.train).forward)(blobs_in,
blobs_out,
name='mySplitOp')
def DistributeCascadeProposals(self, stage):
###用于将proposals分配给各level, 若采用fpn架构的话
"""Distribute proposals to their appropriate FPN levels.
by Zhaowei Cai for Cascade R-CNN
Input blobs:
###输入为第j个stage所输出的调整过的proposals
- proposals_<j> are the decoded proposals from stage j; see
documentation from DecodeBBoxes.
###若在训练阶段使用，还会有以下输入blobs: [roidb, im_info]
If used during training, then the input blobs will also include:
[roidb, im_info] (see GenerateProposalLabels).
Output blobs: [rois_fpn<min>, ..., rois_rpn<max>, rois,
rois_idx_restore]
###输出为：每一个 fpn lvl的proposals, 所有proposals的一个排列(为了
###保留这些proposals在input blob中的初始顺序？？)
- rois_fpn<i> are the RPN proposals for FPN level i
- rois_idx_restore is a permutation on the concatenation of all
rois_fpn<i>, i=min...max, such that when applied the RPN RoIs are
restored to their original order in the input blobs.
###训练阶段还包含：gt框的相关信息，其中bbox_inside_weights表示回归损失公式
###中的Pi*, 即bbox_inside_weights==1, bbox_outside_weights==0即只对正样本
###计算回归损失
If used during training, then the output blobs will also include:
[labels, bbox_targets, bbox_inside_weights, bbox_outside_weights,
mapped_gt_boxes].
"""
stage_name = '_{}'.format(stage)
# Prepare input blobs
blobs_in = ['proposals' + stage_name]
if self.train:
blobs_in += ['roidb', 'im_info']
####下面这一句是干嘛的？？？
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'DistributeCascadeProposalsOp:' + ','.join(
[str(b) for b in blobs_in])
# Prepare output blobs
##准备输出数据blob容器
blobs_out = cascade_rcnn_roi_data.get_cascade_rcnn_blob_names(
stage, is_training=self.train)
###又是这个函数
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
outputs = self.net.Python(
DistributeCascadeProposalsOp(self.train, stage).forward)(blobs_in,
blobs_out,
name=name)
return outputs
def distillLoss(self, blobs_in, blobs_out, temperature):
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
self.net.Python(
distillLossOp(self.train, temperature).forward,
distillLossOp(self.train,
temperature).backwardd)(blobs_in,
blobs_out,
name='distillLoss')
def GenerateRoIsNeedRefine(self):
### IMPORTANT! Unused op!!!
""" Generate a binary label to decide whether the prediction needs
further refinement. And also update the corresponding prediction
here.
Input blobs: ['prn_probs']
- prn_probs is the probability of the mask/keypoint
prediction needs further refinement
If used during training, includes the labels for PredictNeedRefine
and use it as the output. ['prn_labels_int32']
And also adds the related blobs for the specific refinement task,
such as ['refined_masks_int32'].
Output blob: ['roi_needs_refine_int32']
- roi_needs_refine_int32 is a binary label indicates whether
further refinement is needed or not.
And if used during training, also doing inplace-update for the
labels of refinement tasks. Such as update ['refined_masks_int32']
"""
# Prepare input blobs
blobs_in = ['prn_probs']
if self.train:
# adds labels of prn
blobs_in += ['prn_labels_int32']
# adds refinement tasks specific blobs
if cfg.MODEL.REFINE_MASK_ON:
blobs_in += ['refined_masks_int32']
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'GenerateRoIsNeedRefineOp: ' + ','.join(
str(b) for b in blobs_in
)
# Prepare output blobs
blobs_out = ['roi_needs_refine_int32']
if self.train:
# add refinement tasks specific blobs
if cfg.MODEL.REFINE_MASK_ON:
blobs_out += ['refined_masks_int32']
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
# Execute op
# Note that this op will overwrite the label for the specific task
outputs = self.net.Python(
GenerateRoIsNeedRefineOp(self.train).forward
)(blobs_in, blobs_out, name=name)
return outputs[0] # only return the binary label
def SpatialBN(self, blob_in, blob_out, dim_in, **kwargs):
blob_out = blob_out or self.net.NextName()
# Input: input, scale, bias, est_mean, est_inv_var
# Output: output, running_mean, running_inv_var, saved_mean,
# saved_inv_var
# scale: initialize with ones
# bias: initialize with zeros
# est mean: zero
# est var: ones
def init_blob(value, suffix):
return self.param_init_net.ConstantFill([],
blob_out + "_" + suffix,
shape=[dim_in],
value=value)
if self.init_params:
scale, bias = init_blob(1.0, "s"), init_blob(0.0, "b")
running_mean = init_blob(0.0, "rm")
running_inv_var = init_blob(1.0, "riv")
else:
scale = core.ScopedBlobReference(blob_out + '_s',
self.param_init_net)
bias = core.ScopedBlobReference(blob_out + '_b',
self.param_init_net)
running_mean = core.ScopedBlobReference(blob_out + '_rm',
self.param_init_net)
running_inv_var = core.ScopedBlobReference(blob_out + '_riv',
self.param_init_net)
self.params.extend([scale, bias])
self.computed_params.extend([running_mean, running_inv_var])
self.weights.append(scale)
self.biases.append(bias)
blob_outs = [
blob_out, running_mean, running_inv_var, blob_out + "_sm",
blob_out + "_siv"
]
if 'is_test' in kwargs and kwargs['is_test']:
blob_outputs = self.net.SpatialBN(
[blob_in, scale, bias, blob_outs[1], blob_outs[2]], [blob_out],
order=self.order,
**kwargs)
return blob_outputs
else:
blob_outputs = self.net.SpatialBN(
[blob_in, scale, bias, blob_outs[1], blob_outs[2]],
blob_outs,
order=self.order,
**kwargs)
# Return the output
return blob_outputs[0]
def MaskIoUs(self, blobs_in, blob_label, blob_out):
""" Calculate Mask IoUs.
Input blobs: ['mask_probs', 'masks_int32']
Output blobs: ['mask_ious']
"""
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'MaskIoUsOp: ' + ','.join([str(b) for b in blobs_in])
blob_out = core.ScopedBlobReference(blob_out)
output = self.net.Python(
MaskIoUsOp().forward
)(blobs_in, blob_out, name=name)
return output
def DistributeFpnRpnProposals(self, stage_num=2):
"""Distribute proposals from former rcnn heads to their appropriate FPN levels.
An anchor at one FPN level may predict an RoI that will map to another level,
hence the need to redistribute the proposals.
This function assumes standard blob names for input and output blobs.
Input blobs: [rois_2nd/3rd, roi_scores_2nd/3rd]
- rois are the proposals
- roi_probs are the proposal objectness probabilities
see rpn_roi_probs documentation from GenerateProposals.
If used during training, then the input blobs will also include:
[roidb, im_info] (see GenerateProposalLabels).
Output blobs: [rois_fpn<min>, ..., rois_rpn<max>, rois,
rois_idx_restore]
- rois_fpn<i> are the proposals for FPN level i
- rois_idx_restore is a permutation on the concatenation of all
rois_fpn<i>, i=min...max, such that when applied the RPN RoIs are
restored to their original order in the input blobs.
If used during training, then the output blobs will also include:
[labels, bbox_targets, bbox_inside_weights, bbox_outside_weights].
"""
# Prepare input blobs
if stage_num == 2:
rois_names = ['rois_2nd_pre']
elif stage_num == 3:
rois_names = ['rois_3rd_pre']
blobs_in = rois_names
if self.train:
blobs_in += ['roidb', 'im_info']
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'DistributeFpnRpnProposalsOp:' + ','.join(
[str(b) for b in blobs_in]
)
# Prepare output blobs
blobs_out = fast_rcnn_roi_data.get_cascade_fast_rcnn_blob_names(
is_training=self.train, stage_num=stage_num
)
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
outputs = self.net.Python(
DistributeFpnRpnProposalsOp(self.train, stage_num).forward
)(blobs_in, blobs_out, name=name)
return outputs
def get_weightsFastrcnn(self):
blobs_in = ['cls_score_toothbrush']
blobs_out = [
'freeze_cls_score', 'fastrcnn_cls_loss_inside_weights',
'fastrcnn_cls_loss_outside_weights', 'freeze_bbox_pred',
'fastrcnn_reg_loss_inside_weights',
'fastrcnn_reg_loss_outside_weights'
]
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
self.net.Python(get_weightsOpFastrcnn(self.train).forward)(
blobs_in, blobs_out, name='get_weightsOpFastrcnn')
def Conv(self,
blob_in,
blob_out,
dim_in,
dim_out,
kernel,
weight_init=None,
bias_init=None,
**kwargs):
"""Convolution. We intentionally do not provide odd kernel/stride/pad
settings in order to discourage the use of odd cases.
"""
weight_init = weight_init if weight_init else ('XavierFill', {})
bias_init = bias_init if bias_init else ('ConstantFill', {})
blob_out = blob_out or self.net.NextName()
weight_shape = ([dim_out, dim_in, kernel, kernel] if self.order
== "NCHW" else [dim_out, kernel, kernel, dim_in])
if self.init_params:
weight = self.param_init_net.__getattr__(
weight_init[0])([],
blob_out + '_w',
shape=weight_shape,
**weight_init[1])
bias = self.param_init_net.__getattr__(
bias_init[0])([],
blob_out + '_b',
shape=[
dim_out,
],
**bias_init[1])
else:
weight = core.ScopedBlobReference(blob_out + '_w',
self.param_init_net)
bias = core.ScopedBlobReference(blob_out + '_b',
self.param_init_net)
self.params.extend([weight, bias])
self.weights.append(weight)
self.biases.append(bias)
if self.use_cudnn:
kwargs['engine'] = 'CUDNN'
kwargs['exhaustive_search'] = self.cudnn_exhaustive_search
if self.ws_nbytes_limit:
kwargs['ws_nbytes_limit'] = self.ws_nbytes_limit
return self.net.Conv([blob_in, weight, bias],
blob_out,
kernel=kernel,
order=self.order,
**kwargs)
def __init__(self,
model,
input_record,
num_splits,
axis=1,
name='split',
**kwargs):
super(Split, self).__init__(model, name, input_record, **kwargs)
self.axis = axis
# Assume that first dimension is batch, so actual axis in shape is
# axis - 1
axis -= 1
assert axis >= 0
assert isinstance(input_record, schema.Scalar),\
"Incorrect input type. Excpected Scalar, but received: {0}".\
format(input_record)
input_shape = input_record.field_type().shape
assert len(input_shape) >= axis
assert input_shape[axis] % num_splits == 0
output_shape = list(input_shape)
output_shape[axis] = int(output_shape[axis] / num_splits)
data_type = input_record.field_type().base
output_scalars = [
schema.Scalar(
(data_type, output_shape),
core.ScopedBlobReference(
model.net.NextName(self.name + '_output_{}'.format(i))))
for i in range(num_splits)
]
self.output_schema = schema.Tuple(*output_scalars)
def model_build_fun(model, loss_scale):
workspace.FeedBlob(
core.ScopedBlobReference("seq_lengths"),
np.array([self.T] * self.batch_per_device, dtype=np.int32))
model.param_init_net.ConstantFill(
[],
"hidden_init",
value=0.0,
shape=[1, self.batch_per_device, self.hidden_dim])
model.param_init_net.ConstantFill(
[],
"cell_init",
value=0.0,
shape=[1, self.batch_per_device, self.hidden_dim])
output, _last_hidden, _, _last_state, = recurrent.LSTM(
model=model,
input_blob="data",
seq_lengths="seq_lengths",
initial_states=("hidden_init", "cell_init"),
dim_in=self.input_dim,
dim_out=self.hidden_dim,
scope="partest",
)
# A silly loss function
loss = model.AveragedLoss(
model.Sub([output, "target"], "dist"),
"loss",
)
loss = model.Scale(loss, "loss_scaled", scale=loss_scale)
return [loss]
def GenerateProposalLabels(self, blobs_in):
"""Op for generating training labels for RPN proposals. This is used
when training RPN jointly with Fast/Mask R-CNN (as in end-to-end
Faster R-CNN training).
blobs_in:
- 'rpn_rois': 2D tensor of RPN proposals output by GenerateProposals
- 'roidb': roidb entries that will be labeled
- 'im_info': See GenerateProposals doc.
blobs_out:
- (variable set of blobs): returns whatever blobs are required for
training the model. It does this by querying the data loader for
the list of blobs that are needed.
"""
name = 'GenerateProposalLabelsOp:' + ','.join(
[str(b) for b in blobs_in])
# The list of blobs is not known before run-time because it depends on
# the specific model being trained. Query the data loader to get the
# list of output blob names.
blobs_out = roi_data.fast_rcnn.get_fast_rcnn_blob_names(
is_training=self.train)
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
self.net.Python(GenerateProposalLabelsOp().forward)(blobs_in,
blobs_out,
name=name)
return blobs_out
def __init__(self, model, input_record, axis=1, name='concat', **kwargs):
super(Concat, self).__init__(model, name, input_record, **kwargs)
self.axis = axis
assert isinstance(input_record, schema.Struct),\
"Incorrect input type. Excpected Struct, but received: {0}".\
format(input_record)
shapes = []
for field_name, field_type in input_record.fields.items():
assert isinstance(field_type, schema.Scalar),\
"Incorrect input type. Excpected Scalar, but received: {0}".\
format(field_type)
# Assume that first dimension is batch, so actual axis in shape is
# axis - 1
assert len(field_type.field_type().shape) >= axis,\
"Concat expects that limited dimensions of the input tensor"
shapes.append(list(field_type.field_type().shape))
concat_dim = 0
for shape in shapes:
concat_dim += shape[axis - 1]
shape[axis - 1] = 0
assert shape == shapes[0],\
"Shapes {0} and {1} are not compatible for Concat".\
format(shape, shapes[0])
output_dims = shapes[0]
output_dims[axis - 1] = concat_dim
self.output_schema = schema.Scalar(
(np.float32, output_dims),
core.ScopedBlobReference(model.net.NextName(self.name +
'_output')))
def DistributeCascadeProposals(self, stage):
"""Distribute proposals to their appropriate FPN levels.
by Zhaowei Cai for Cascade R-CNN
将proposals映射到合适的fpn层
Input blobs:
- proposals_<j> are the decoded proposals from stage j; see
documentation from DecodeBBoxes.
If used during training, then the input blobs will also include:
[roidb, im_info] (see GenerateProposalLabels).
Output blobs: [rois_fpn<min>, ..., rois_rpn<max>, rois,
rois_idx_restore]
- rois_fpn<i> are the RPN proposals for FPN level i
- rois_idx_restore is a permutation on the concatenation of all
rois_fpn<i>, i=min...max, such that when applied the RPN RoIs are
restored to their original order in the input blobs.
If used during training, then the output blobs will also include:
[labels, bbox_targets, bbox_inside_weights, bbox_outside_weights,
mapped_gt_boxes].
"""
stage_name = '_{}'.format(stage)
# Prepare input blobs
blobs_in = ['proposals' + stage_name]
if self.train:
blobs_in += ['roidb', 'im_info']
blobs_in = [core.ScopedBlobReference(b) for b in blobs_in]
name = 'DistributeCascadeProposalsOp:' + ','.join(
[str(b) for b in blobs_in]
)
# Prepare output blobs
blobs_out = cascade_rcnn_roi_data.get_cascade_rcnn_blob_names(
stage, is_training=self.train
)
blobs_out = [core.ScopedBlobReference(b) for b in blobs_out]
outputs = self.net.Python(
DistributeCascadeProposalsOp(self.train, stage).forward
)(blobs_in, blobs_out, name=name)
return outputs
