def add_wsl_fpn_retinanet_losses(model):
loss_gradients = {}
gradients, losses = [], []
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
for lvl in range(k_min, k_max + 1):
suffix = 'fpn{}'.format(lvl)
roi_prob_name = 'retnet_cls_prob_' + suffix
cls_prob_name = 'cls_prob_' + suffix
cross_entropy_name = 'cross_entropy_' + suffix
cls_loss_name = 'cls_loss_' + suffix
model.net.RoIScorePool(roi_prob_name,
cls_prob_name,
num_classes=model.num_classes - 1)
cross_entropy = model.net.CrossEntropyWithLogits(
[cls_prob_name, 'cls_labels'], [cross_entropy_name])
cls_loss = model.net.AveragedLoss([cross_entropy_name],
[cls_loss_name])
gradients.append(cls_loss)
losses.append(cls_loss_name)
loss_gradients.update(blob_utils.get_loss_gradients(model, gradients))
model.AddLosses(losses)
return loss_gradients
def add_single_scale_rpn_losses(model):
"""Add losses for a single scale RPN model (i.e., no FPN)."""
# Spatially narrow the full-sized RPN label arrays to match the feature map
# shape
model.net.SpatialNarrowAs(
['rpn_labels_int32_wide', 'rpn_cls_logits'], 'rpn_labels_int32'
)
for key in ('targets', 'inside_weights', 'outside_weights'):
model.net.SpatialNarrowAs(
['rpn_bbox_' + key + '_wide', 'rpn_bbox_pred'], 'rpn_bbox_' + key
)
loss_rpn_cls = model.net.SigmoidCrossEntropyLoss(
['rpn_cls_logits', 'rpn_labels_int32'],
'loss_rpn_cls',
scale=model.GetLossScale()
)
loss_rpn_bbox = model.net.SmoothL1Loss(
[
'rpn_bbox_pred', 'rpn_bbox_targets', 'rpn_bbox_inside_weights',
'rpn_bbox_outside_weights'
],
'loss_rpn_bbox',
beta=1. / 9.,
scale=model.GetLossScale()
)
loss_gradients = blob_utils.get_loss_gradients(
model, [loss_rpn_cls, loss_rpn_bbox]
)
model.AddLosses(['loss_rpn_cls', 'loss_rpn_bbox'])
return loss_gradients
def add_single_scale_rpn_losses(model):
model.net.SpatialNarrowAs(
['rpn_labels_int32_wide', 'rpn_cls_logits'], 'rpn_labels_int32'
)
for key in ('targets', 'inside_weights', 'outside_weights'):
model.net.SpatialNarrowAs(
['rpn_bbox_' + key + '_wide', 'rpn_bbox_pred'], 'rpn_bbox_' + key
)
loss_rpn_cls = model.net.SigmoidCrossEntropyLoss(
['rpn_cls_logits', 'rpn_labels_int32'],
'loss_rpn_cls',
scale=model.GetLossScale()
)
loss_rpn_bbox = model.net.SmoothL1Loss(
[
'rpn_bbox_pred', 'rpn_bbox_tragets', 'rpn_bbox_inside_weights',
'rpn_bbox_outside_weights'
],
'loss_rpn_bbox',
beta=1. / 9.,
scale=model.GetLossScale()
)
loss_gradients = blob_utils.get_loss_gradients(
model, [loss_rpn_cls, loss_rpn_bbox]
)
model.AddLosses(['loss_rpn_cls', 'loss_rpn_bbox'])
return loss_gradients
def add_mask_res_loss(model, blob_mask_res):
model.net.NCHW2NHWC(blob_mask_res, 'seg_score_NHWC')
model.Reshape('seg_score_NHWC',
['seg_score_reshape', 'seg_score_old_shape'],
shape=[-1, model.num_classes])
model.Reshape('seg_gt_label',
['seg_gt_label_reshape', 'seg_gt_label_shape'],
shape=[
-1,
])
probs_human, loss_human = model.net.SoftmaxWithLoss(
['seg_score_reshape', 'seg_gt_label_reshape'],
['probs_human', 'loss_human'],
scale=1. / cfg.NUM_GPUS)
'''
# focal loss
loss_human, probs_human = model.net.SoftmaxFocalLoss(
[blob_mask_res, 'seg_gt_label', 'normalizer'], ['loss_human', 'probs_human'],
scale=model.GetLossScale(), gamma=2., num_classes=model.num_classes
)
'''
loss_gradients = blob_utils.get_loss_gradients(model, [loss_human])
model.AddLosses('loss_human')
return loss_gradients
def _add_instance_level_classifier(model, blob_in, dim_in):
from detectron.utils.c2 import const_fill
from detectron.utils.c2 import gauss_fill
def negateGrad(inputs, outputs):
outputs[0].feed(inputs[0].data)
def grad_negateGrad(inputs, outputs):
scale = cfg.TRAIN.DA_INS_GRL_WEIGHT
grad_output = inputs[-1]
outputs[0].reshape(grad_output.shape)
outputs[0].data[...] = -1.0*scale*grad_output.data
model.GradientScalerLayer([blob_in], ['dc_grl'], -1.0*cfg.TRAIN.DA_INS_GRL_WEIGHT)
model.FC('dc_grl', 'dc_ip1', dim_in, 1024,
weight_init=gauss_fill(0.01), bias_init=const_fill(0.0))
model.Relu('dc_ip1', 'dc_relu_1')
model.Dropout('dc_relu_1', 'dc_drop_1', ratio=0.5, is_test=False)
model.FC('dc_drop_1', 'dc_ip2', 1024, 1024,
weight_init=gauss_fill(0.01), bias_init=const_fill(0.0))
model.Relu('dc_ip2', 'dc_relu_2')
model.Dropout('dc_relu_2', 'dc_drop_2', ratio=0.5, is_test=False)
dc_ip3 = model.FC('dc_drop_2', 'dc_ip3', 1024, 1,
weight_init=gauss_fill(0.05), bias_init=const_fill(0.0))
loss_gradient = None
if model.train:
dc_loss = model.net.SigmoidCrossEntropyLoss(
[dc_ip3, 'dc_label'],
'loss_dc',
scale=model.GetLossScale()
)
loss_gradient = blob_utils.get_loss_gradients(model, [dc_loss])
model.AddLosses('loss_dc')
return loss_gradient
def add_fast_rcnn_losses(model):
"""Add losses for RoI classification and bounding box regression."""
if cfg.TRAIN.DOMAIN_ADAPTATION:
model.MaskingInput(['cls_score', 'label_mask'], ['source_cls_score'])
model.MaskingInput(['bbox_pred', 'label_mask'], ['source_bbox_pred'])
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['source_cls_score', 'source_labels_int32'],
['cls_prob', 'loss_cls'],
scale=model.GetLossScale())
loss_bbox = model.net.SmoothL1Loss([
'source_bbox_pred', 'source_bbox_targets',
'source_bbox_inside_weights', 'source_bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale())
model.Accuracy(['cls_prob', 'source_labels_int32'], 'accuracy_cls')
else:
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['cls_score', 'labels_int32'], ['cls_prob', 'loss_cls'],
scale=model.GetLossScale())
loss_bbox = model.net.SmoothL1Loss([
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale())
model.Accuracy(['cls_prob', 'labels_int32'], 'accuracy_cls')
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
model.AddLosses(['loss_cls', 'loss_bbox'])
model.AddMetrics('accuracy_cls')
return loss_gradients
def _single_gpu_build_func(model):
loss = {"loss": None}
p = model.Conv("data_stage2", 'conv1_stage2', 40, 20, kernel=3, pad=1, stride=1, no_bias=1,
weight_init=(cfg.MRCNN.CONV_INIT, {'std': 0.001}))
p = model.AffineChannel(p, 'conv1_bn_stage2', dim=20, inplace=True)
p = model.Relu(p, p)
human_fc = model.Conv(p, 'conv2_stage2', 20, 20, kernel=1, pad=0, stride=1,
weight_init=(cfg.MRCNN.CONV_INIT, {'std': 0.001}))
if not model.train:
model.net.NCHW2NHWC(human_fc, 'seg_score_NHWC_stage2')
model.net.Softmax('seg_score_NHWC_stage2', 'probs_human_NHWC_stage2', axis=3)
model.net.NHWC2NCHW('probs_human_NHWC_stage2', 'probs_human_NCHW_stage2')
loss_gradient = None
if model.train:
model.net.NCHW2NHWC(human_fc, 'seg_score_NHWC_stage2')
model.Reshape('seg_score_NHWC_stage2', ['seg_score_reshape_stage2', 'seg_score_old_shape_stage2'],
shape=[-1, model.num_classes])
model.Reshape('gt_label_stage2', ['gt_label_reshape_stage2', 'gt_label_shape_stage2'], shape=[-1, ])
probs_human, loss_human = model.net.SoftmaxWithLoss(
['seg_score_reshape_stage2', 'gt_label_reshape_stage2'],
['probs_human_stage2', 'loss_human_stage2'],
scale=1. / cfg.NUM_GPUS)
loss_gradient = blob_utils.get_loss_gradients(model, [loss_human])
model.AddLosses('loss_human_stage2')
loss['loss'] = loss_gradient
if model.train:
loss_gradients = {}
for lg in loss.values():
if lg is not None:
loss_gradients.update(lg)
return loss_gradients
else:
return None
def add_cluster_rcnn_losses(model):
"""Add losses for cluster RoI classification and bounding box regression."""
loss_scalar = 1.0
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['cluster_cls_score', 'cluster_labels_int32'],
['cluster_cls_prob', 'loss_cluster_cls'],
scale=model.GetLossScale() * loss_scalar)
loss_bbox = model.net.SmoothL1Loss([
'cluster_bbox_pred', 'cluster_bbox_targets',
'cluster_bbox_inside_weights', 'cluster_bbox_outside_weights'
],
'loss_cluster_bbox',
scale=model.GetLossScale() *
loss_scalar)
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
model.Accuracy(['cluster_cls_prob', 'cluster_labels_int32'],
'accuracy_cluster_cls')
model.AddLosses(['loss_cluster_cls', 'loss_cluster_bbox'])
model.AddMetrics('accuracy_cluster_cls')
bbox_reg_weights = cfg.MODEL.BBOX_REG_WEIGHTS
model.AddBBoxAccuracy([
'cluster_bbox_pred', 'cluster_rois', 'cluster_labels_int32',
'mapped_gt_cluster_boxes'
], ['cluster_bbox_iou', 'cluster_bbox_iou_pre'], bbox_reg_weights)
model.AddMetrics(['cluster_bbox_iou', 'cluster_bbox_iou_pre'])
return loss_gradients
def add_cmil_losses(model, prefix=''):
loss_gradients = {}
model.net.RoIIoU([prefix + 'rois'], [prefix + 'rois_iou'])
import uuid
uu = uuid.uuid4().int % 10000
K = 2
for k in range(1, K + 1):
if k == 1:
input_blobs = [
prefix + 'cmil_rois_pred', prefix + 'rois_iou', 'labels_oh',
prefix + 'cls_prob'
]
else:
input_blobs = [
prefix + 'cls_prob' + str(k - 1), prefix + 'rois_iou',
'labels_oh', prefix + 'cls_prob'
]
model.net.RoILabel(
input_blobs,
[
prefix + 'rois_labels_int32' + str(k),
prefix + 'rois_weight' + str(k)
],
display=int(1280 / cfg.NUM_GPUS),
uuid=uu,
fg_thresh=0.6,
bg_thresh_hi=0.4,
bg_thresh_lo=0.1,
num_pos=32,
num_neg=96,
)
cls_prob, loss_cls = model.net.SoftmaxWithLossN(
[
prefix + 'cls_score' + str(k), prefix + 'rois_labels_int32' +
str(k), prefix + 'rois_weight' + str(k)
],
[prefix + 'cls_prob' + str(k), prefix + 'loss_cls' + str(k)],
# scale=model.GetLossScale(),
)
if cfg.WSL.MEAN_LOSS:
lg = blob_utils.get_loss_gradients(model, [loss_cls])
else:
lg = get_loss_gradients_weighted(model, [loss_cls],
1. * (cfg.MODEL.NUM_CLASSES - 1))
loss_gradients.update(lg)
model.Accuracy([
prefix + 'cls_prob' + str(k), prefix + 'rois_labels_int32' + str(k)
], prefix + 'accuracy_cls' + str(k))
model.AddLosses([prefix + 'loss_cls' + str(k)])
model.AddMetrics(prefix + 'accuracy_cls' + str(k))
return loss_gradients
def add_fpn_retinanet_losses(model):
loss_gradients = {}
gradients, losses = [], []
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
model.AddMetrics(['retnet_fg_num', 'retnet_bg_num'])
# ==========================================================================
# bbox regression loss - SelectSmoothL1Loss for multiple anchors at a location
# ==========================================================================
for lvl in range(k_min, k_max + 1):
suffix = 'fpn{}'.format(lvl)
bbox_loss = model.net.SelectSmoothL1Loss(
[
'retnet_bbox_pred_' + suffix, 'retnet_roi_bbox_targets_' +
suffix, 'retnet_roi_fg_bbox_locs_' + suffix, 'retnet_fg_num'
],
'retnet_loss_bbox_' + suffix,
beta=cfg.RETINANET.BBOX_REG_BETA,
scale=model.GetLossScale() * cfg.RETINANET.BBOX_REG_WEIGHT)
gradients.append(bbox_loss)
losses.append('retnet_loss_bbox_' + suffix)
# ==========================================================================
# cls loss - depends on softmax/sigmoid outputs
# ==========================================================================
for lvl in range(k_min, k_max + 1):
suffix = 'fpn{}'.format(lvl)
cls_lvl_logits = 'retnet_cls_pred_' + suffix
if not cfg.RETINANET.SOFTMAX:
cls_focal_loss = model.net.SigmoidFocalLoss(
[
cls_lvl_logits, 'retnet_cls_labels_' + suffix,
'retnet_fg_num'
], ['fl_{}'.format(suffix)],
gamma=cfg.RETINANET.LOSS_GAMMA,
alpha=cfg.RETINANET.LOSS_ALPHA,
scale=model.GetLossScale(),
num_classes=model.num_classes - 1)
gradients.append(cls_focal_loss)
losses.append('fl_{}'.format(suffix))
else:
cls_focal_loss, gated_prob = model.net.SoftmaxFocalLoss(
[
cls_lvl_logits, 'retnet_cls_labels_' + suffix,
'retnet_fg_num'
], ['fl_{}'.format(suffix), 'retnet_prob_{}'.format(suffix)],
gamma=cfg.RETINANET.LOSS_GAMMA,
alpha=cfg.RETINANET.LOSS_ALPHA,
scale=model.GetLossScale(),
num_classes=model.num_classes)
gradients.append(cls_focal_loss)
losses.append('fl_{}'.format(suffix))
loss_gradients.update(blob_utils.get_loss_gradients(model, gradients))
model.AddLosses(losses)
return loss_gradients
def add_body_uv_losses(model):
"""Add DensePose body UV specific losses."""
# Pool estimated IUV points via bilinear interpolation.
for name in ['U', 'V', 'Index_UV']:
model.PoolPointsInterp([
name + '_estimated' if name in ['U', 'V'] else name,
'body_uv_coords_xy'
], ['interp_' + name])
# Compute spatial softmax normalized probabilities, after which
# cross-entropy loss is computed for semantic parts classification.
probs_AnnIndex, loss_AnnIndex = model.net.SpatialSoftmaxWithLoss(
['AnnIndex', 'body_uv_parts', 'body_uv_parts_weights'],
['probs_AnnIndex', 'loss_AnnIndex'],
scale=model.GetLossScale() * cfg.BODY_UV_RCNN.INDEX_WEIGHTS)
# Softmax loss for surface patch classification.
probs_I_points, loss_I_points = model.net.SoftmaxWithLoss(
['interp_Index_UV', 'body_uv_I_points'],
['probs_I_points', 'loss_I_points'],
scale=model.GetLossScale() * cfg.BODY_UV_RCNN.PART_WEIGHTS,
spatial=0)
## Smooth L1 loss for each patch-specific UV coordinates regression.
# Reshape U,V blobs of both interpolated and ground-truth to compute
# summarized (instead of averaged) SmoothL1Loss.
loss_UV = list()
model.net.Reshape(['body_uv_point_weights'],
['UV_point_weights', 'body_uv_point_weights_shape'],
shape=(1, -1, cfg.BODY_UV_RCNN.NUM_PATCHES + 1))
for name in ['U', 'V']:
# Reshape U/V coordinates of both interpolated points and ground-truth
# points from (#points, #patches) to (1, #points, #patches).
model.net.Reshape(
['body_uv_' + name + '_points'],
[name + '_points', 'body_uv_' + name + '_points_shape'],
shape=(1, -1, cfg.BODY_UV_RCNN.NUM_PATCHES + 1))
model.net.Reshape(
['interp_' + name],
['interp_' + name + '_reshaped', 'interp_' + name + 'shape'],
shape=(1, -1, cfg.BODY_UV_RCNN.NUM_PATCHES + 1))
# Compute summarized SmoothL1Loss of all points.
loss_UV.append(
model.net.SmoothL1Loss([
'interp_' + name + '_reshaped', name + '_points',
'UV_point_weights', 'UV_point_weights'
],
'loss_' + name + '_points',
scale=model.GetLossScale() *
cfg.BODY_UV_RCNN.POINT_REGRESSION_WEIGHTS))
# Add all losses to compute gradients
loss_gradients = blob_utils.get_loss_gradients(
model, [loss_AnnIndex, loss_I_points] + loss_UV)
# Update model training losses
model.AddLosses([
'loss_' + name
for name in ['AnnIndex', 'I_points', 'U_points', 'V_points']
])
return loss_gradients
def add_mask_rcnn_losses(model, blob_mask):
"""Add Mask R-CNN specific losses."""
loss_mask = model.net.SigmoidCrossEntropyLoss([blob_mask, 'masks_int32'],
'loss_mask',
scale=model.GetLossScale() *
cfg.MRCNN.WEIGHT_LOSS_MASK)
loss_gradients = blob_utils.get_loss_gradients(model, [loss_mask])
model.AddLosses('loss_mask')
return loss_gradients
def add_cascade_rcnn_losses(model, stage):
####增加损失项：用于roi分类和边界框回归
"""Add losses for RoI classification and bounding box regression."""
stage_name = "_{}".format(stage)
####
if cfg.CASCADE_RCNN.SCALE_LOSS:
####loss_scalar这一项好像定义了每一阶段的训练损失占总损失的比重
loss_scalar = cfg.CASCADE_RCNN.STAGE_WEIGHTS[stage - 1]
else:
loss_scalar = 1.0
### 获得该stage的分类分数和分类损失，先经过softmax，再做损失
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
["cls_score" + stage_name, "labels_int32" + stage_name],
["cls_prob" + stage_name, "loss_cls" + stage_name],
scale=model.GetLossScale() * loss_scalar,
)
### 得到bbox回归损失，为smoothl1-loss
loss_bbox = model.net.SmoothL1Loss(
[
"bbox_pred" + stage_name,
"bbox_targets" + stage_name,
"bbox_inside_weights" + stage_name,
"bbox_outside_weights" + stage_name,
],
"loss_bbox" + stage_name,
scale=model.GetLossScale() * loss_scalar,
)
### 由两项损失计算损失梯度
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
### 得到模型精度
model.Accuracy(
["cls_prob" + stage_name, "labels_int32" + stage_name],
"accuracy_cls" + stage_name,
)
### 对模型增加损失（分类和回归损失）
model.AddLosses(["loss_cls" + stage_name, "loss_bbox" + stage_name])
### 对模型增加指标：分类精确度
model.AddMetrics("accuracy_cls" + stage_name)
###得到该阶段bbox回归参数的权重
bbox_reg_weights = cfg.CASCADE_RCNN.BBOX_REG_WEIGHTS[stage - 1]
###对模型增加bbox回归精度
model.AddBBoxAccuracy(
[
"bbox_pred" + stage_name,
"rois" + stage_name,
"labels_int32" + stage_name,
"mapped_gt_boxes" + stage_name,
],
["bbox_iou" + stage_name, "bbox_iou" + stage_name + "_pre"],
bbox_reg_weights,
)
###对模型增加指标：Bbox iou值
model.AddMetrics(
["bbox_iou" + stage_name, "bbox_iou" + stage_name + "_pre"])
return loss_gradients
def add_mask_rcnn_losses(model, blob_mask):
"""Add Mask R-CNN specific losses."""
loss_mask = model.net.SigmoidCrossEntropyLoss(
[blob_mask, 'masks_int32'],
'loss_mask',
scale=model.GetLossScale() * cfg.MRCNN.WEIGHT_LOSS_MASK
)
loss_gradients = blob_utils.get_loss_gradients(model, [loss_mask])
model.AddLosses('loss_mask')
return loss_gradients
def add_fast_rcnn_losses(model):
"""Add losses for RoI classification and bounding box regression."""
if cfg.TRAIN.DOMAIN_ADAPTATION:
scores = 'cls_score'
if cfg.TRAIN.PADA:
scores = 'pada_' + scores
model.MaskingInput([scores, 'label_mask'], ['source_cls_score'])
model.MaskingInput(['bbox_pred', 'label_mask'], ['source_bbox_pred'])
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['source_cls_score', 'source_labels_int32'], ['cls_prob', 'loss_cls'],
scale=model.GetLossScale()
)
loss_bbox = model.net.SmoothL1Loss(
[
'source_bbox_pred', 'source_bbox_targets', 'source_bbox_inside_weights',
'source_bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale()
)
model.Accuracy(['cls_prob', 'source_labels_int32'], 'accuracy_cls')
def update_conf_matrix(inputs,outputs):
cls_prob = inputs[0].data
labels = inputs[1].data
# print(cls_prob.shape)
# print(labels.shape)
model.class_weight_db.update_confusion_matrix(cls_prob,labels)
model.net.Python(update_conf_matrix)(['cls_prob','source_labels_int32'],[],name='UpdateConfusionMatrix')
else:
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['cls_score', 'labels_int32'], ['cls_prob', 'loss_cls'],
scale=model.GetLossScale()
)
loss_bbox = model.net.SmoothL1Loss(
[
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale()
)
model.Accuracy(['cls_prob', 'labels_int32'], 'accuracy_cls')
loss_gradients = blob_utils.get_loss_gradients(model, [loss_cls, loss_bbox])
model.AddLosses(['loss_cls', 'loss_bbox'])
model.AddMetrics('accuracy_cls')
return loss_gradients
def add_mlp_losses(model):
"""Add losses for classification """
cls_prob, loss_cls = model.net.SoftmaxWithLoss(['logits', 'labels_int32'],
['cls_prob', 'loss_cls'],
scale=model.GetLossScale())
loss_gradients = blob_utils.get_loss_gradients(model, [loss_cls])
model.Accuracy(['cls_prob', 'labels_int32'], 'accuracy_cls')
model.AddLosses(['loss_cls'])
model.AddMetrics('accuracy_cls')
return loss_gradients
def add_fpn_rpn_losses(model):
"""Add RPN on FPN specific losses."""
loss_gradients = {}
for lvl in range(cfg.FPN.RPN_MIN_LEVEL, cfg.FPN.RPN_MAX_LEVEL + 1):
slvl = str(lvl)
# Spatially narrow the full-sized RPN label arrays to match the feature map
# shape
model.net.SpatialNarrowAs(
['rpn_labels_int32_wide_fpn' + slvl, 'rpn_cls_logits_fpn' + slvl],
'rpn_labels_int32_fpn' + slvl)
for key in ('targets', 'inside_weights', 'outside_weights'):
model.net.SpatialNarrowAs([
'rpn_bbox_' + key + '_wide_fpn' + slvl,
'rpn_bbox_pred_fpn' + slvl
], 'rpn_bbox_' + key + '_fpn' + slvl)
if cfg.RPN.FOCAL_LOSS:
model.AddMetrics(['rpn_fg_num', 'rpn_bg_num'])
loss_rpn_cls_fpn = model.net.SigmoidFocalLoss(
[
'rpn_cls_logits_fpn' + slvl, 'rpn_labels_int32_fpn' + slvl,
'rpn_fg_num'
],
'loss_rpn_cls_fpn' + slvl,
gamma=cfg.RPN.LOSS_GAMMA,
alpha=cfg.RPN.LOSS_ALPHA,
num_class=1,
scale=(model.GetLossScale() / cfg.TRAIN.RPN_BATCH_SIZE_PER_IM /
cfg.TRAIN.IMS_PER_BATCH))
else:
loss_rpn_cls_fpn = model.net.SigmoidCrossEntropyLoss(
['rpn_cls_logits_fpn' + slvl, 'rpn_labels_int32_fpn' + slvl],
'loss_rpn_cls_fpn' + slvl,
normalize=0,
scale=(model.GetLossScale() / cfg.TRAIN.RPN_BATCH_SIZE_PER_IM /
cfg.TRAIN.IMS_PER_BATCH))
# Normalization by (1) RPN_BATCH_SIZE_PER_IM and (2) IMS_PER_BATCH is
# handled by (1) setting bbox outside weights and (2) SmoothL1Loss
# normalizes by IMS_PER_BATCH
loss_rpn_bbox_fpn = model.net.SmoothL1Loss(
[
'rpn_bbox_pred_fpn' + slvl, 'rpn_bbox_targets_fpn' + slvl,
'rpn_bbox_inside_weights_fpn' + slvl,
'rpn_bbox_outside_weights_fpn' + slvl
],
'loss_rpn_bbox_fpn' + slvl,
beta=1. / 9.,
scale=model.GetLossScale(),
)
loss_gradients.update(
blob_utils.get_loss_gradients(
model, [loss_rpn_cls_fpn, loss_rpn_bbox_fpn]))
model.AddLosses(
['loss_rpn_cls_fpn' + slvl, 'loss_rpn_bbox_fpn' + slvl])
return loss_gradients
def _add_consistency_loss(model, blob_img_in, img_dim_in, blob_ins_in, ins_dim_in):
def expand_as(inputs, outputs):
img_prob = inputs[0].data
ins_prob = inputs[1].data
import numpy as np
mean_da_conv = np.mean(img_prob, (1,2,3))
repeated_da_conv = np.expand_dims(np.repeat(
mean_da_conv, ins_prob.shape[0]//2), axis=1)
outputs[0].feed(repeated_da_conv)
def grad_expand_as(inputs, outputs):
import numpy as np
img_prob = inputs[0].data
ins_prob = inputs[1].data
grad_output = inputs[3]
grad_input = outputs[0]
grad_input.reshape(inputs[0].shape)
unit = grad_output.shape[0]//2
grad_o = grad_output.data[...]
grad_i = np.zeros(inputs[0].shape)
for i in range(inputs[0].shape[0]):
grad_i[i] = np.sum(grad_o[i*unit:(i+1)*unit, 0])*np.ones(grad_i[i].shape).astype(np.float32)/(img_prob.shape[1]*img_prob.shape[2]*img_prob.shape[3])
grad_input.data[...] = grad_i
model.GradientScalerLayer([blob_img_in], ['da_grl_copy'], 1.0*cfg.TRAIN.DA_IMG_GRL_WEIGHT)
model.ConvShared('da_grl_copy', 'da_conv_1_copy', img_dim_in, 512, kernel=1, pad=0, stride=1, weight='da_conv_1_w', bias='da_conv_1_b')
model.Relu('da_conv_1_copy', 'da_conv_1_copy')
model.ConvShared('da_conv_1_copy', 'da_conv_2_copy', 512, 1, kernel=1, pad=0, stride=1, weight='da_conv_2_w', bias='da_conv_2_b')
model.net.Sigmoid('da_conv_2_copy', 'img_probs')
model.GradientScalerLayer([blob_ins_in], ['dc_grl_copy'], 1.0*cfg.TRAIN.DA_INS_GRL_WEIGHT)
model.FCShared('dc_grl_copy', 'dc_ip1_copy', ins_dim_in, 1024,
weight='dc_ip1_w', bias='dc_ip1_b')
model.Relu('dc_ip1_copy', 'dc_relu_1_copy')
model.FCShared('dc_relu_1_copy', 'dc_ip2_copy', 1024, 1024,
weight='dc_ip2_w', bias='dc_ip2_b')
model.Relu('dc_ip2_copy', 'dc_relu_2_copy')
model.FCShared('dc_relu_2_copy', 'dc_ip3_copy', 1024, 1,
weight='dc_ip3_w', bias='dc_ip3_b')
model.net.Sigmoid('dc_ip3_copy', "ins_probs")
loss_gradient = None
if model.train:
model.net.Python(f=expand_as, grad_f=grad_expand_as, grad_input_indices=[0], grad_output_indices=[0])(
['img_probs', 'ins_probs'], ['repeated_img_probs'])
dist = model.net.L1Distance(['repeated_img_probs', 'ins_probs'], ['consistency_dist'])
# dist = model.net.SquaredL2Distance(['repeated_img_probs', 'ins_probs'], ['consistency_dist'])
loss_consistency = model.net.AveragedLoss(dist, 'loss_consistency')
loss_gradient = blob_utils.get_loss_gradients(
model, [loss_consistency])
model.AddLosses('loss_consistency')
return loss_gradient
def _add_instance_level_classifier(model, blob_in, dim_in, spatial_scale):
from detectron.utils.c2 import const_fill
from detectron.utils.c2 import gauss_fill
# def negateGrad(inputs, outputs):
# outputs[0].feed(inputs[0].data)
# def grad_negateGrad(inputs, outputs):
# scale = cfg.TRAIN.DA_INS_GRL_WEIGHT
# grad_output = inputs[-1]
# outputs[0].reshape(grad_output.shape)
# outputs[0].data[...] = -1.0*scale*grad_output.data
model.RoIFeatureTransform(
blob_in,
'da_pool5',
blob_rois='da_rois',
method=cfg.FAST_RCNN.ROI_XFORM_METHOD,
resolution=7,
sampling_ratio=cfg.FAST_RCNN.ROI_XFORM_SAMPLING_RATIO,
spatial_scale=spatial_scale
)
model.FCShared('da_pool5', 'da_fc6', dim_in * 7 * 7, 4096,
weight='fc6_w', bias='fc6_b')
model.Relu('da_fc6', 'da_fc6')
model.FCShared('da_fc6', 'da_fc7', 4096, 4096,
weight='fc7_w', bias='fc7_b')
da_blobs = model.Relu('da_fc7', 'da_fc7')
model.GradientScalerLayer([da_blobs], ['dc_grl'], -1.0*cfg.TRAIN.DA_INS_GRL_WEIGHT)
model.FC('dc_grl', 'dc_ip1', 4096, 1024,
weight_init=gauss_fill(0.01), bias_init=const_fill(0.0))
model.Relu('dc_ip1', 'dc_relu_1')
model.Dropout('dc_relu_1', 'dc_drop_1', ratio=0.5, is_test=False)
model.FC('dc_drop_1', 'dc_ip2', 1024, 1024,
weight_init=gauss_fill(0.01), bias_init=const_fill(0.0))
model.Relu('dc_ip2', 'dc_relu_2')
model.Dropout('dc_relu_2', 'dc_drop_2', ratio=0.5, is_test=False)
dc_ip3 = model.FC('dc_drop_2', 'dc_ip3', 1024, 1,
weight_init=gauss_fill(0.05), bias_init=const_fill(0.0))
if cfg.TRAIN.PADA:
dc_ip3 = model.PADAbyGradientWeightingLayerD(dc_ip3,'pada_dc_ip3','pada_roi_weights')
loss_gradient = None
if model.train:
dc_loss = model.net.SigmoidCrossEntropyLoss(
[dc_ip3, 'dc_label'],
'loss_dc',
scale=model.GetLossScale()
)
loss_gradient = blob_utils.get_loss_gradients(model, [dc_loss])
model.AddLosses('loss_dc')
return loss_gradient, da_blobs, 4096
def add_apm_losses1(model):
add_cls_pred('rois_pred', 'cls_prob', model)
cls_prob_softmax, loss_cls = model.net.SoftmaxWithLoss(
['cls_prob', 'labels_int32'], ['cls_prob_softmax', 'loss_cls'],
scale=model.GetLossScale())
loss_gradients = blob_utils.get_loss_gradients(model, [loss_cls])
model.Accuracy(['cls_prob_softmax', 'labels_int32'], 'accuracy_cls')
model.AddLosses(['loss_cls'])
model.AddMetrics('accuracy_cls')
return loss_gradients
def add_bpm_attr_losses(model):
"""Add losses for RoI classification and bounding box regression."""
# View prediction loss
attr_cls_prob, attr_loss_cls = model.net.SoftmaxWithLoss(
['attr_fc', 'attr_labels_int32', 'attr_weight'],
['attr_cls_prob', 'attr_loss_cls'],
scale=model.GetLossScale())
model.Accuracy(['attr_cls_prob', 'attr_labels_int32'], 'attr_accuracy_cls')
model.AddMetrics('attr_accuracy_cls')
loss_gradients = blob_utils.get_loss_gradients(model, [attr_loss_cls])
model.AddLosses([attr_loss_cls])
loss_gradients.update(add_bpm_losses(model))
return loss_gradients
def add_pcl_losses(model, prefix=''):
loss_gradients = {}
K = 3
for k in range(1, K + 1):
if k == 1:
input_blobs = [
prefix + 'rois',
prefix + 'rois_pred',
'labels_oh',
prefix + 'cls_prob' + str(k),
prefix + 'cls_prob',
]
else:
input_blobs = [
prefix + 'rois',
prefix + 'cls_prob' + str(k - 1),
'labels_oh',
prefix + 'cls_prob' + str(k),
prefix + 'cls_prob',
]
output_blobs = [
prefix + 'labels' + str(k),
prefix + 'cls_loss_weights' + str(k),
prefix + 'gt_assignment' + str(k),
prefix + 'pc_labels' + str(k),
prefix + 'pc_probs' + str(k),
prefix + 'pc_count' + str(k),
prefix + 'img_cls_loss_weights' + str(k),
prefix + 'im_labels_real' + str(k),
]
name = 'PCL:' + str(k)
model.net.Python(PCLOp().forward)(input_blobs, output_blobs, name=name)
loss_cls = model.net.PCLLoss(
[
prefix + 'cls_prob' + str(k),
] + output_blobs,
[prefix + 'loss_cls' + str(k)],
)
lg = blob_utils.get_loss_gradients(model, [loss_cls])
loss_gradients.update(lg)
model.AddLosses([prefix + 'loss_cls' + str(k)])
return loss_gradients
def add_fpn_rpn_losses(model):
"""Add RPN on FPN specific losses."""
# 添加损失
loss_gradients = {}
for lvl in range(cfg.FPN.RPN_MIN_LEVEL, cfg.FPN.RPN_MAX_LEVEL + 1):
slvl = str(lvl)
# Spatially narrow the full-sized RPN label arrays to match the feature map
# shape
# TODO(zzdxfei) work here
model.net.SpatialNarrowAs(
['rpn_labels_int32_wide_fpn' + slvl, 'rpn_cls_logits_fpn' + slvl],
'rpn_labels_int32_fpn' + slvl)
for key in ('targets', 'inside_weights', 'outside_weights'):
model.net.SpatialNarrowAs([
'rpn_bbox_' + key + '_wide_fpn' + slvl,
'rpn_bbox_pred_fpn' + slvl
], 'rpn_bbox_' + key + '_fpn' + slvl)
# 分类损失
loss_rpn_cls_fpn = model.net.SigmoidCrossEntropyLoss(
['rpn_cls_logits_fpn' + slvl, 'rpn_labels_int32_fpn' + slvl],
'loss_rpn_cls_fpn' + slvl,
normalize=0,
scale=(model.GetLossScale() / cfg.TRAIN.RPN_BATCH_SIZE_PER_IM /
cfg.TRAIN.IMS_PER_BATCH))
# Normalization by (1) RPN_BATCH_SIZE_PER_IM and (2) IMS_PER_BATCH is
# handled by (1) setting bbox outside weights and (2) SmoothL1Loss
# normalizes by IMS_PER_BATCH
# 包围盒回归损失
loss_rpn_bbox_fpn = model.net.SmoothL1Loss(
[
'rpn_bbox_pred_fpn' + slvl, 'rpn_bbox_targets_fpn' + slvl,
'rpn_bbox_inside_weights_fpn' + slvl,
'rpn_bbox_outside_weights_fpn' + slvl
],
'loss_rpn_bbox_fpn' + slvl,
beta=1. / 9.,
scale=model.GetLossScale(),
)
# 梯度更新
loss_gradients.update(
blob_utils.get_loss_gradients(
model, [loss_rpn_cls_fpn, loss_rpn_bbox_fpn]))
model.AddLosses(
['loss_rpn_cls_fpn' + slvl, 'loss_rpn_bbox_fpn' + slvl])
return loss_gradients
def add_fast_rcnn_losses(model):
"""Add losses for RoI classification and bounding box regression."""
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['cls_score', 'labels_int32'], ['cls_prob', 'loss_cls'],
scale=model.GetLossScale())
loss_bbox = model.net.SmoothL1Loss([
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale())
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
model.Accuracy(['cls_prob', 'labels_int32'], 'accuracy_cls')
model.AddLosses(['loss_cls', 'loss_bbox'])
model.AddMetrics('accuracy_cls')
return loss_gradients
def add_cascade_rcnn_losses(model, stage):
"""Add losses for RoI classification and bounding box regression."""
stage_name = "_{}".format(stage)
if cfg.CASCADE_RCNN.SCALE_LOSS:
loss_scalar = cfg.CASCADE_RCNN.STAGE_WEIGHTS[stage - 1]
else:
loss_scalar = 1.0
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
["cls_score" + stage_name, "labels_int32" + stage_name],
["cls_prob" + stage_name, "loss_cls" + stage_name],
scale=model.GetLossScale() * loss_scalar,
)
loss_bbox = model.net.SmoothL1Loss(
[
"bbox_pred" + stage_name,
"bbox_targets" + stage_name,
"bbox_inside_weights" + stage_name,
"bbox_outside_weights" + stage_name,
],
"loss_bbox" + stage_name,
scale=model.GetLossScale() * loss_scalar,
)
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
model.Accuracy(
["cls_prob" + stage_name, "labels_int32" + stage_name],
"accuracy_cls" + stage_name,
)
model.AddLosses(["loss_cls" + stage_name, "loss_bbox" + stage_name])
model.AddMetrics("accuracy_cls" + stage_name)
bbox_reg_weights = cfg.CASCADE_RCNN.BBOX_REG_WEIGHTS[stage - 1]
model.AddBBoxAccuracy(
[
"bbox_pred" + stage_name,
"rois" + stage_name,
"labels_int32" + stage_name,
"mapped_gt_boxes" + stage_name,
],
["bbox_iou" + stage_name, "bbox_iou" + stage_name + "_pre"],
bbox_reg_weights,
)
model.AddMetrics(
["bbox_iou" + stage_name, "bbox_iou" + stage_name + "_pre"])
return loss_gradients
def _add_image_level_classifier(model, blob_in, dim_in, spatial_scale_in):
from detectron.utils.c2 import const_fill
from detectron.utils.c2 import gauss_fill
def negateGrad(inputs, outputs):
outputs[0].feed(inputs[0].data)
def grad_negateGrad(inputs, outputs):
scale = cfg.TRAIN.DA_IMG_GRL_WEIGHT
grad_output = inputs[-1]
outputs[0].reshape(grad_output.shape)
outputs[0].data[...] = -1.0 * scale * grad_output.data
model.GradientScalerLayer([blob_in], ['da_grl'],
-1.0 * cfg.TRAIN.DA_IMG_GRL_WEIGHT)
model.Conv('da_grl',
'da_conv_1',
dim_in,
512,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.001),
bias_init=const_fill(0.0))
model.Relu('da_conv_1', 'da_conv_1')
model.Conv('da_conv_1',
'da_conv_2',
512,
1,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.001),
bias_init=const_fill(0.0))
if model.train:
model.net.SpatialNarrowAs(['da_label_wide', 'da_conv_2'], 'da_label')
loss_da = model.net.SigmoidCrossEntropyLoss(['da_conv_2', 'da_label'],
'loss_da',
scale=model.GetLossScale())
loss_gradient = blob_utils.get_loss_gradients(model, [loss_da])
model.AddLosses('loss_da')
return loss_gradient
else:
return None
def add_fast_rcnn_losses(model):
"""Add losses for RoI classification and bounding box regression."""
cls_prob, loss_cls = model.net.SoftmaxWithLoss(
['cls_score', 'labels_int32'], ['cls_prob', 'loss_cls'],
scale=model.GetLossScale()
)
loss_bbox = model.net.SmoothL1Loss(
[
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale()
)
loss_gradients = blob_utils.get_loss_gradients(model, [loss_cls, loss_bbox])
model.Accuracy(['cls_prob', 'labels_int32'], 'accuracy_cls')
model.AddLosses(['loss_cls', 'loss_bbox'])
model.AddMetrics('accuracy_cls')
return loss_gradients
def add_fast_rcnn_weighted_losses(model):
"""Add losses for RoI classification and bounding box regression."""
cls_prob1, loss_cls1 = model.net.SoftmaxWithLoss(
['cls_score', 'labels_int32'], ['cls_prob1', 'loss_cls1'],
scale=model.GetLossScale())
cls_prob2 = model.net.Sigmoid('cls_score', 'cls_prob2')
model.LabelCrossEntropy(['cls_prob2', 'labels_int32'],
'sigmoid_cross_entropy')
loss_cls2 = model.AveragedLoss('sigmoid_cross_entropy', 'loss_cls2')
loss_cls = model.net.Python(weighted_loss_forward, weighted_loss_backward)(
['loss_cls1', 'weight_cls1', 'loss_cls2', 'weight_cls2'], 'loss_cls')
cls_prob = model.Mean(['cls_prob1', 'cls_prob2'], 'cls_prob')
loss_bbox1 = model.net.SmoothL1Loss([
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox1',
scale=model.GetLossScale())
loss_bbox2 = model.net.MSELoss([
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox2',
scale=model.GetLossScale())
loss_bbox = model.net.Python(
weighted_loss_forward, weighted_loss_backward)(
['loss_bbox1', 'weight_bbox1', 'loss_bbox2', 'weight_bbox2'],
'loss_bbox')
#loss_gradients = blob_utils.get_loss_gradients(model, [loss_cls1, loss_cls2, loss_bbox1, loss_bbox2])
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
model.Accuracy(['cls_prob', 'labels_int32'], 'accuracy_cls')
#model.Accuracy(['cls_prob2', 'labels_int32'], 'accuracy_cls2')
#model.AddLosses(['loss_cls1', 'loss_bbox1'])
#model.AddLosses(['loss_cls2', 'loss_bbox2'])
model.AddLosses(['loss_cls', 'loss_bbox'])
model.AddMetrics([
'accuracy_cls', 'weight_cls1', 'weight_cls2', 'weight_bbox1',
'weight_bbox2'
])
#model.AddMetrics('accuracy_cls2')
return loss_gradients
def add_mask_rcnn_losses(model, blob_mask):
"""Add Mask R-CNN specific losses."""
if cfg.MRCNN.DP_CASCADE_MASK_ON:
return add_cascade_dp_mask_losses(model, blob_mask)
'''
probs_mask, loss_mask = model.net.SpatialSoftmaxWithLoss( \
[blob_mask, 'mask_labels','mask_weights'],\
['probs_mask','loss_mask'], \
scale=cfg.BODY_UV_RCNN.INDEX_WEIGHTS / cfg.NUM_GPUS)
'''
loss_mask = model.net.SigmoidCrossEntropyLoss([blob_mask, 'masks_int32'],
'loss_mask',
scale=model.GetLossScale() *
cfg.MRCNN.WEIGHT_LOSS_MASK)
losses = ['loss_mask']
losses_to_gradients = [loss_mask]
if cfg.MRCNN.BBOX_CASCADE_MASK_ON:
loss_inter_mask = model.net.SigmoidCrossEntropyLoss(
['inter_person_mask', 'inter_masks_int32'],
'loss_inter_mask',
scale=model.GetLossScale() * cfg.MRCNN.WEIGHT_LOSS_MASK)
losses += ['loss_inter_mask']
losses_to_gradients += [loss_inter_mask]
if cfg.MRCNN.USE_CLS_EMBS:
#loss_mask_emb = model.net.SigmoidCrossEntropyLoss(
# ['mask_emb_logits', 'mask_emb_labels'],
# 'loss_mask_emb',
# scale=model.GetLossScale() * cfg.MRCNN.WEIGHT_LOSS_MASK
#)
model.net.Concat(['mask_emb_fg_labels', 'mask_emb_bg_labels'],
['mask_emb_labels', 'mask_emb_label_shape'],
axis=0)
mask_emb_prob, loss_mask_emb = model.net.SoftmaxWithLoss(
['mask_emb_logits', 'mask_emb_labels'],
['mask_emb_prob', 'loss_mask_emb'],
scale=model.GetLossScale())
losses += ['loss_mask_emb']
losses_to_gradients += [loss_mask_emb]
loss_gradients = blob_utils.get_loss_gradients(model, losses_to_gradients)
model.AddLosses(losses)
return loss_gradients
def add_fast_rcnn_losses(model):
"""Add losses for RoI classification and bounding box regression."""
# cls_prob, loss_cls = model.net.SoftmaxWithLoss(
# ['cls_score', 'labels_int32'], ['cls_prob', 'loss_cls'],
# scale=model.GetLossScale()
# )
# loss_cls = model.net.SigmoidFocalLoss(
# [
# 'cls_score', 'labels_int32',
# 'fg_num'
# ],
# ['loss_cls'],
# gamma=cfg.RETINANET.LOSS_GAMMA,
# alpha=cfg.RETINANET.LOSS_ALPHA,
# scale=model.GetLossScale(),
# num_classes=model.num_classes - 1
# )
loss_cls, cls_prob = model.net.SoftmaxFocalLoss(
['cls_score_reshape', 'labels_int32', 'fg_num'],
['loss_cls', 'cls_prob'],
gamma=cfg.RETINANET.LOSS_GAMMA,
alpha=cfg.RETINANET.LOSS_ALPHA,
scale=model.GetLossScale(),
num_classes=model.num_classes)
# cls_prob_reshape = model.net.Reshape(
# ['cls_prob'], ['cls_prob_reshape', 'prob_old_shape'],
# shape=(-1, 0))
loss_bbox = model.net.SmoothL1Loss([
'bbox_pred', 'bbox_targets', 'bbox_inside_weights',
'bbox_outside_weights'
],
'loss_bbox',
scale=model.GetLossScale())
loss_gradients = blob_utils.get_loss_gradients(model,
[loss_cls, loss_bbox])
# model.Accuracy(['cls_prob_reshape', 'labels_int32'], 'accuracy_cls')
model.AddLosses(['loss_cls', 'loss_bbox'])
# model.AddMetrics('accuracy_cls')
return loss_gradients
def add_single_scale_rpn_losses(model):
"""Add losses for a single scale RPN model (i.e., no FPN)."""
# Spatially narrow the full-sized RPN label arrays to match the feature map
# shape
model.net.SpatialNarrowAs(
['rpn_labels_int32_wide', 'rpn_cls_logits'], 'rpn_labels_int32'
)
for key in ('targets', 'inside_weights', 'outside_weights'):
model.net.SpatialNarrowAs(
['rpn_bbox_' + key + '_wide', 'rpn_bbox_pred'], 'rpn_bbox_' + key
)
if cfg.RPN.FOCAL_LOSS:
model.AddMetrics(['rpn_fg_num', 'rpn_bg_num'])
loss_rpn_cls_fpn = model.net.SigmoidFocalLoss(
['rpn_cls_logits', 'rpn_labels_int32', 'rpn_fg_num'],
'loss_rpn_cls',
gamma=cfg.RPN.LOSS_GAMMA,
alpha=cfg.RPN.LOSS_ALPHA,
num_class=1,
scale= model.GetLossScale()
)
else:
loss_rpn_cls = model.net.SigmoidCrossEntropyLoss(
['rpn_cls_logits', 'rpn_labels_int32'],
'loss_rpn_cls',
scale=model.GetLossScale()
)
loss_rpn_bbox = model.net.SmoothL1Loss(
[
'rpn_bbox_pred', 'rpn_bbox_targets', 'rpn_bbox_inside_weights',
'rpn_bbox_outside_weights'
],
'loss_rpn_bbox',
beta=1. / 9.,
scale=model.GetLossScale()
)
loss_gradients = blob_utils.get_loss_gradients(
model, [loss_rpn_cls, loss_rpn_bbox]
)
model.AddLosses(['loss_rpn_cls', 'loss_rpn_bbox'])
return loss_gradients
def add_cascade_dp_mask_losses(model, blob_mask):
model.net.Reshape( ['body_mask_labels'], \
['body_mask_labels_reshaped', 'body_uv_mask_labels_old_shape'], \
shape=(-1, cfg.BODY_UV_RCNN.HEATMAP_SIZE , cfg.BODY_UV_RCNN.HEATMAP_SIZE))
model.net.Reshape( ['body_uv_ann_weights'], \
['body_uv_mask_weights_reshaped', 'body_uv_mask_weights_old_shape'], \
shape=( -1 , cfg.BODY_UV_RCNN.HEATMAP_SIZE , cfg.BODY_UV_RCNN.HEATMAP_SIZE))
# intermediate loss
probs_inter_mask, loss_inter_mask = model.net.SpatialSoftmaxWithLoss( \
['inter_person_mask', 'body_mask_labels_reshaped','body_uv_mask_weights_reshaped'],\
['probs_inter_mask','loss_inter_mask'], \
scale=cfg.BODY_UV_RCNN.INDEX_WEIGHTS / cfg.NUM_GPUS)
# final mask loss
probs_mask, loss_mask = model.net.SpatialSoftmaxWithLoss( \
['person_mask', 'body_mask_labels_reshaped','body_uv_mask_weights_reshaped'],\
['probs_mask','loss_mask'], \
scale=cfg.BODY_UV_RCNN.INDEX_WEIGHTS / cfg.NUM_GPUS)
loss_gradients = blob_utils.get_loss_gradients(model, [loss_inter_mask, loss_mask])
model.AddLosses(['loss_inter_mask', 'loss_mask'])
return loss_gradients
def add_keypoint_losses(model):
"""Add Mask R-CNN keypoint specific losses."""
# Reshape input from (N, K, H, W) to (NK, HW)
model.net.Reshape(
['kps_score'], ['kps_score_reshaped', '_kps_score_old_shape'],
shape=(-1, cfg.KRCNN.HEATMAP_SIZE * cfg.KRCNN.HEATMAP_SIZE)
)
# Softmax across **space** (woahh....space!)
# Note: this is not what is commonly called "spatial softmax"
# (i.e., softmax applied along the channel dimension at each spatial
# location); This is softmax applied over a set of spatial locations (i.e.,
# each spatial location is a "class").
kps_prob, loss_kps = model.net.SoftmaxWithLoss(
['kps_score_reshaped', 'keypoint_locations_int32', 'keypoint_weights'],
['kps_prob', 'loss_kps'],
scale=cfg.KRCNN.LOSS_WEIGHT / cfg.NUM_GPUS,
spatial=0
)
if not cfg.KRCNN.NORMALIZE_BY_VISIBLE_KEYPOINTS:
# Discussion: the softmax loss above will average the loss by the sum of
# keypoint_weights, i.e. the total number of visible keypoints. Since
# the number of visible keypoints can vary significantly between
# minibatches, this has the effect of up-weighting the importance of
# minibatches with few visible keypoints. (Imagine the extreme case of
# only one visible keypoint versus N: in the case of N, each one
# contributes 1/N to the gradient compared to the single keypoint
# determining the gradient direction). Instead, we can normalize the
# loss by the total number of keypoints, if it were the case that all
# keypoints were visible in a full minibatch. (Returning to the example,
# this means that the one visible keypoint contributes as much as each
# of the N keypoints.)
model.StopGradient(
'keypoint_loss_normalizer', 'keypoint_loss_normalizer'
)
loss_kps = model.net.Mul(
['loss_kps', 'keypoint_loss_normalizer'], 'loss_kps_normalized'
)
loss_gradients = blob_utils.get_loss_gradients(model, [loss_kps])
model.AddLosses(loss_kps)
return loss_gradients
def add_fpn_retinanet_losses(model):
loss_gradients = {}
gradients, losses = [], []
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
model.AddMetrics(['retnet_fg_num', 'retnet_bg_num'])
# ==========================================================================
# bbox regression loss - SelectSmoothL1Loss for multiple anchors at a location
# ==========================================================================
for lvl in range(k_min, k_max + 1):
suffix = 'fpn{}'.format(lvl)
bbox_loss = model.net.SelectSmoothL1Loss(
[
'retnet_bbox_pred_' + suffix,
'retnet_roi_bbox_targets_' + suffix,
'retnet_roi_fg_bbox_locs_' + suffix, 'retnet_fg_num'
],
'retnet_loss_bbox_' + suffix,
beta=cfg.RETINANET.BBOX_REG_BETA,
scale=model.GetLossScale() * cfg.RETINANET.BBOX_REG_WEIGHT
)
gradients.append(bbox_loss)
losses.append('retnet_loss_bbox_' + suffix)
# ==========================================================================
# cls loss - depends on softmax/sigmoid outputs
# ==========================================================================
for lvl in range(k_min, k_max + 1):
suffix = 'fpn{}'.format(lvl)
cls_lvl_logits = 'retnet_cls_pred_' + suffix
if not cfg.RETINANET.SOFTMAX:
cls_focal_loss = model.net.SigmoidFocalLoss(
[
cls_lvl_logits, 'retnet_cls_labels_' + suffix,
'retnet_fg_num'
],
['fl_{}'.format(suffix)],
gamma=cfg.RETINANET.LOSS_GAMMA,
alpha=cfg.RETINANET.LOSS_ALPHA,
scale=model.GetLossScale(),
num_classes=model.num_classes - 1
)
gradients.append(cls_focal_loss)
losses.append('fl_{}'.format(suffix))
else:
cls_focal_loss, gated_prob = model.net.SoftmaxFocalLoss(
[
cls_lvl_logits, 'retnet_cls_labels_' + suffix,
'retnet_fg_num'
],
['fl_{}'.format(suffix), 'retnet_prob_{}'.format(suffix)],
gamma=cfg.RETINANET.LOSS_GAMMA,
alpha=cfg.RETINANET.LOSS_ALPHA,
scale=model.GetLossScale(),
num_classes=model.num_classes
)
gradients.append(cls_focal_loss)
losses.append('fl_{}'.format(suffix))
loss_gradients.update(blob_utils.get_loss_gradients(model, gradients))
model.AddLosses(losses)
return loss_gradients
def add_body_uv_losses(model, pref=''):
## Reshape for GT blobs.
model.net.Reshape( ['body_uv_X_points'], ['X_points_reshaped'+pref, 'X_points_shape'+pref], shape=( -1 ,1 ) )
model.net.Reshape( ['body_uv_Y_points'], ['Y_points_reshaped'+pref, 'Y_points_shape'+pref], shape=( -1 ,1 ) )
model.net.Reshape( ['body_uv_I_points'], ['I_points_reshaped'+pref, 'I_points_shape'+pref], shape=( -1 ,1 ) )
model.net.Reshape( ['body_uv_Ind_points'], ['Ind_points_reshaped'+pref, 'Ind_points_shape'+pref], shape=( -1 ,1 ) )
## Concat Ind,x,y to get Coordinates blob.
model.net.Concat( ['Ind_points_reshaped'+pref,'X_points_reshaped'+pref, \
'Y_points_reshaped'+pref],['Coordinates'+pref,'Coordinate_Shapes'+pref ], axis = 1 )
##
### Now reshape UV blobs, such that they are 1x1x(196*NumSamples)xNUM_PATCHES
## U blob to
##
model.net.Reshape(['body_uv_U_points'], \
['U_points_reshaped'+pref, 'U_points_old_shape'+pref],\
shape=(-1,cfg.BODY_UV_RCNN.NUM_PATCHES+1,196))
model.net.Transpose(['U_points_reshaped'+pref] ,['U_points_reshaped_transpose'+pref],axes=(0,2,1) )
model.net.Reshape(['U_points_reshaped_transpose'+pref], \
['U_points'+pref, 'U_points_old_shape2'+pref], \
shape=(1,1,-1,cfg.BODY_UV_RCNN.NUM_PATCHES+1))
## V blob
##
model.net.Reshape(['body_uv_V_points'], \
['V_points_reshaped'+pref, 'V_points_old_shape'+pref],\
shape=(-1,cfg.BODY_UV_RCNN.NUM_PATCHES+1,196))
model.net.Transpose(['V_points_reshaped'+pref] ,['V_points_reshaped_transpose'+pref],axes=(0,2,1) )
model.net.Reshape(['V_points_reshaped_transpose'+pref], \
['V_points'+pref, 'V_points_old_shape2'+pref], \
shape=(1,1,-1,cfg.BODY_UV_RCNN.NUM_PATCHES+1))
###
## UV weights blob
##
model.net.Reshape(['body_uv_point_weights'], \
['Uv_point_weights_reshaped'+pref, 'Uv_point_weights_old_shape'+pref],\
shape=(-1,cfg.BODY_UV_RCNN.NUM_PATCHES+1,196))
model.net.Transpose(['Uv_point_weights_reshaped'+pref] ,['Uv_point_weights_reshaped_transpose'+pref],axes=(0,2,1) )
model.net.Reshape(['Uv_point_weights_reshaped_transpose'+pref], \
['Uv_point_weights'+pref, 'Uv_point_weights_old_shape2'+pref], \
shape=(1,1,-1,cfg.BODY_UV_RCNN.NUM_PATCHES+1))
#####################
### Pool IUV for points via bilinear interpolation.
model.PoolPointsInterp(['U_estimated','Coordinates'+pref], ['interp_U'+pref])
model.PoolPointsInterp(['V_estimated','Coordinates'+pref], ['interp_V'+pref])
model.PoolPointsInterp(['Index_UV'+pref,'Coordinates'+pref], ['interp_Index_UV'+pref])
## Reshape interpolated UV coordinates to apply the loss.
model.net.Reshape(['interp_U'+pref], \
['interp_U_reshaped'+pref, 'interp_U_shape'+pref],\
shape=(1, 1, -1 , cfg.BODY_UV_RCNN.NUM_PATCHES+1))
model.net.Reshape(['interp_V'+pref], \
['interp_V_reshaped'+pref, 'interp_V_shape'+pref],\
shape=(1, 1, -1 , cfg.BODY_UV_RCNN.NUM_PATCHES+1))
###
### Do the actual labels here !!!!
model.net.Reshape( ['body_uv_ann_labels'], \
['body_uv_ann_labels_reshaped' +pref, 'body_uv_ann_labels_old_shape'+pref], \
shape=(-1, cfg.BODY_UV_RCNN.HEATMAP_SIZE , cfg.BODY_UV_RCNN.HEATMAP_SIZE))
model.net.Reshape( ['body_uv_ann_weights'], \
['body_uv_ann_weights_reshaped' +pref, 'body_uv_ann_weights_old_shape'+pref], \
shape=( -1 , cfg.BODY_UV_RCNN.HEATMAP_SIZE , cfg.BODY_UV_RCNN.HEATMAP_SIZE))
###
model.net.Cast( ['I_points_reshaped'+pref], ['I_points_reshaped_int'+pref], to=core.DataType.INT32)
### Now add the actual losses
## The mask segmentation loss (dense)
probs_seg_AnnIndex, loss_seg_AnnIndex = model.net.SpatialSoftmaxWithLoss( \
['AnnIndex'+pref, 'body_uv_ann_labels_reshaped'+pref,'body_uv_ann_weights_reshaped'+pref],\
['probs_seg_AnnIndex'+pref,'loss_seg_AnnIndex'+pref], \
scale=cfg.BODY_UV_RCNN.INDEX_WEIGHTS / cfg.NUM_GPUS)
## Point Patch Index Loss.
probs_IndexUVPoints, loss_IndexUVPoints = model.net.SoftmaxWithLoss(\
['interp_Index_UV'+pref,'I_points_reshaped_int'+pref],\
['probs_IndexUVPoints'+pref,'loss_IndexUVPoints'+pref], \
scale=cfg.BODY_UV_RCNN.PART_WEIGHTS / cfg.NUM_GPUS, spatial=0)
## U and V point losses.
loss_Upoints = model.net.SmoothL1Loss( \
['interp_U_reshaped'+pref, 'U_points'+pref, \
'Uv_point_weights'+pref, 'Uv_point_weights'+pref], \
'loss_Upoints'+pref, \
scale=cfg.BODY_UV_RCNN.POINT_REGRESSION_WEIGHTS / cfg.NUM_GPUS)
loss_Vpoints = model.net.SmoothL1Loss( \
['interp_V_reshaped'+pref, 'V_points'+pref, \
'Uv_point_weights'+pref, 'Uv_point_weights'+pref], \
'loss_Vpoints'+pref, scale=cfg.BODY_UV_RCNN.POINT_REGRESSION_WEIGHTS / cfg.NUM_GPUS)
## Add the losses.
loss_gradients = blob_utils.get_loss_gradients(model, \
[ loss_Upoints, loss_Vpoints, loss_seg_AnnIndex, loss_IndexUVPoints])
model.losses = list(set(model.losses + \
['loss_Upoints'+pref , 'loss_Vpoints'+pref , \
'loss_seg_AnnIndex'+pref ,'loss_IndexUVPoints'+pref]))
return loss_gradients