def forward(self, pred_cls_list, rpn_num_prob_list, pred_reg_list,
anchors_list, rpn_iou_list, boxes, im_info):
all_anchors_list = [
F.concat([a, i * F.ones([a.shape[0], 1]).to(a.device)], axis=1)
for i, a in enumerate(anchors_list)
]
all_anchors_final = F.concat(all_anchors_list, axis=0)
rpn_bbox_offset_final = F.concat(pred_reg_list, axis=1)
rpn_cls_prob_final = F.concat(pred_cls_list, axis=1)
rpn_iou_prob_final = F.concat(rpn_iou_list, axis=1)
rpn_num_per_points_final = F.concat(rpn_num_prob_list, axis=1)
rpn_labels, rpn_target_boxes = rpn_anchor_target_opr(
boxes, im_info, all_anchors_final)
ious_target = self.anchor_iou_target_opr(boxes, im_info,
all_anchors_final,
rpn_bbox_offset_final)
n = rpn_labels.shape[0]
target_boxes = rpn_target_boxes.reshape(n, -1, 4)
rpn_cls_prob_final = rpn_cls_prob_final.reshape(n, -1, 1)
offsets_final = rpn_bbox_offset_final.reshape(n, -1, 4)
rpn_labels = rpn_labels.transpose(2, 0, 1)
a, b = rpn_labels[0], rpn_labels[1]
ignores = b - F.equal(a, 0).astype(np.float32) * F.equal(b, 0).astype(
np.float32)
labels = F.stack([a, ignores], axis=2).reshape(n, -1)
cls_loss = sigmoid_cross_entropy_retina(rpn_cls_prob_final,
labels,
alpha=config.focal_loss_alpha,
gamma=config.focal_loss_gamma)
rpn_bbox_loss = smooth_l1_loss_retina(offsets_final, target_boxes,
labels)
rpn_labels = labels.reshape(n, -1, 2)
rpn_iou_loss = iou_l1_loss(rpn_iou_prob_final, ious_target, rpn_labels)
# whether one anchor produce one proposal or two.
nlabels = ((labels.reshape(n, -1, 2) > 0).sum(2)).flatten() - 1
c = rpn_num_per_points_final.shape[2]
num_per_anchor = rpn_num_per_points_final.reshape(-1, c)
rpn_num_per_points_final = rpn_num_per_points_final.reshape(-1, c)
nlabels = nlabels.reshape(-1)
rpn_num_loss = softmax_loss(rpn_num_per_points_final, nlabels)
loss_dict = {}
loss_dict['rpn_cls_loss'] = cls_loss
loss_dict['rpn_bbox_loss'] = 2 * rpn_bbox_loss
loss_dict['rpn_iou_loss'] = 2 * rpn_iou_loss
loss_dict['rpn_num_loss'] = rpn_num_loss
return loss_dict
def forward(self, features, im_info, boxes=None):
# prediction
pred_cls_score_list = []
pred_bbox_offsets_list = []
for x in features:
t = F.relu(self.rpn_conv(x))
pred_cls_score_list.append(self.rpn_cls_score(t))
pred_bbox_offsets_list.append(self.rpn_bbox_offsets(t))
# get anchors
all_anchors_list = []
fm_stride = 2**(len(features) + 1)
for fm in features:
layer_anchors = self.anchors_generator(fm, fm_stride)
fm_stride = fm_stride // 2
all_anchors_list.append(layer_anchors)
# sample from the predictions
rpn_rois, rpn_probs = find_top_rpn_proposals(self.training,
pred_bbox_offsets_list,
pred_cls_score_list,
all_anchors_list, im_info)
if self.training:
rpn_labels, rpn_bbox_targets = fpn_anchor_target(
boxes, im_info, all_anchors_list)
#rpn_labels = rpn_labels.astype(np.int32)
pred_cls_score, pred_bbox_offsets = fpn_rpn_reshape(
pred_cls_score_list, pred_bbox_offsets_list)
# rpn loss
valid_masks = rpn_labels >= 0
valid_inds = mask_to_inds(valid_masks)
objectness_loss = softmax_loss(pred_cls_score.ai[valid_inds],
rpn_labels.ai[valid_inds])
#objectness_loss = objectness_loss * valid_masks
pos_masks = rpn_labels > 0
localization_loss = smooth_l1_loss(pred_bbox_offsets,
rpn_bbox_targets,
config.rpn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rpn_cls = objectness_loss.sum() * normalizer
loss_rpn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict['loss_rpn_cls'] = loss_rpn_cls
loss_dict['loss_rpn_loc'] = loss_rpn_loc
return rpn_rois, loss_dict
else:
return rpn_rois
def emd_loss(p_b0, p_c0, p_b1, p_c1, targets, labels):
pred_box = F.concat([p_b0, p_b1], axis=1).reshape(-1, p_b0.shapeof()[-1])
pred_score = F.concat([p_c0, p_c1], axis=1).reshape(-1, p_c0.shapeof()[-1])
targets = targets.reshape(-1, 4)
labels = labels.reshape(-1)
fg_masks = F.greater(labels, 0)
non_ignore_masks = F.greater_equal(labels, 0)
# loss for regression
loss_box_reg = smooth_l1_loss(pred_box, targets,
config.rcnn_smooth_l1_beta)
# loss for classification
loss_cls = softmax_loss(pred_score, labels)
loss = loss_cls * non_ignore_masks + loss_box_reg * fg_masks
loss = loss.reshape(-1, 2).sum(axis=1)
return loss.reshape(-1, 1)
def compute_regular_loss(self, prob, bbox_targets, labels):
offsets, cls_scores = prob[:, :-self.n], prob[:, -self.n:]
n = offsets.shape[0]
offsets = offsets.reshape(n, -1, 4)
cls_loss = softmax_loss(cls_scores, labels)
bbox_loss = smooth_l1_loss_rcnn_opr(offsets, bbox_targets, labels,
config.rcnn_smooth_l1_beta)
bbox_loss = bbox_loss.sum() / F.maximum((labels > 0).sum(), 1)
loss = {}
loss['{}_cls_loss'.format(self.name)] = cls_loss
loss['{}_bbox_loss'.format(self.name)] = bbox_loss
return loss
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
pool_features, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align', labels,
bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for regression
labels = labels.astype(np.int32).reshape(-1)
# mulitple class to one
pos_masks = labels > 0
pred_delta = pred_delta.reshape(-1, config.num_classes, 4)
indexing_label = (labels * pos_masks).reshape(-1, 1)
indexing_label = indexing_label.broadcast((labels.shapeof()[0], 4))
pred_delta = F.indexing_one_hot(pred_delta, indexing_label, 1)
localization_loss = smooth_l1_loss(pred_delta, bbox_targets,
config.rcnn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
# loss for classification
valid_masks = labels >= 0
objectness_loss = softmax_loss(pred_cls, labels)
objectness_loss = objectness_loss * valid_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rcnn_cls = objectness_loss.sum() * normalizer
loss_rcnn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict['loss_rcnn_cls'] = loss_rcnn_cls
loss_dict['loss_rcnn_loc'] = loss_rcnn_loc
return loss_dict
else:
pred_scores = F.softmax(pred_cls)[:, 1:].reshape(-1, 1)
pred_delta = pred_delta[:, 4:].reshape(-1, 4)
target_shape = (rcnn_rois.shapeof()[0], config.num_classes - 1, 4)
base_rois = F.add_axis(rcnn_rois[:, 1:5],
1).broadcast(target_shape).reshape(-1, 4)
pred_bbox = restore_bbox(base_rois, pred_delta, True)
pred_bbox = F.concat([pred_bbox, pred_scores], axis=1)
return pred_bbox
def forward(self, fpn_fms, proposals, labels=None, bbox_targets=None):
# input p2-p5
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
#pool_features = roi_pooler(fpn_fms, proposals, stride, (7, 7), "ROIAlignV2")
pool_features, proposals, labels, bbox_targets = roi_pool(
fpn_fms, proposals, stride, (7, 7), 'roi_align',
labels, bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for regression
labels = labels.astype(np.int32).reshape(-1)
# mulitple class to one
pos_masks = labels > 0
localization_loss = smooth_l1_loss(
pred_delta,
bbox_targets,
config.rcnn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
# loss for classification
valid_masks = labels >= 0
objectness_loss = softmax_loss(
pred_cls,
labels)
objectness_loss = objectness_loss * valid_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rcnn_cls = objectness_loss.sum() * normalizer
loss_rcnn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict[self.stage_name + '_cls'] = loss_rcnn_cls
loss_dict[self.stage_name + '_loc'] = loss_rcnn_loc
pred_bbox = restore_bbox(proposals[:, 1:5], pred_delta, True)
pred_proposals = F.zero_grad(F.concat([proposals[:, 0].reshape(-1, 1), pred_bbox], axis=1))
return pred_proposals, loss_dict
else:
pred_scores = F.softmax(pred_cls)[:, 1].reshape(-1, 1)
pred_bbox = restore_bbox(proposals[:, 1:5], pred_delta, True)
pred_proposals = F.concat([proposals[:, 0].reshape(-1, 1), pred_bbox], axis=1)
return pred_proposals, pred_scores
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:]
fpn_fms.reverse()
stride = [4, 8, 16, 32]
poo5, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align',
labels, bbox_targets)
poo5 = F.flatten(poo5, start_axis=1)
fc1 = F.relu(self.fc1(poo5))
fc2 = F.relu(self.fc2(fc1))
cls_scores = self.cls(fc2)
pred_boxes = self.bbox(fc2)
# a = self.a(fc2)
# b = self.b(fc2)
# prob = F.stack([a, b], axis=1).reshape(-1, a.shape[1])
prob = F.concat([pred_boxes, cls_scores], axis=1)
if self.training:
# emd_loss = self.compute_gemini_loss(prob, bbox_targets, labels)
bbox_targets, labels = bbox_targets.reshape(-1, 4), labels.flatten()
cls_loss = softmax_loss(cls_scores, labels)
pred_boxes = pred_boxes.reshape(-1, self.n, 4)
bbox_loss = smooth_l1_loss_rcnn(pred_boxes, bbox_targets, labels, \
config.rcnn_smooth_l1_beta)
loss_dict = {}
loss_dict['cls_loss'] = cls_loss
loss_dict['bbox_loss'] = bbox_loss
return loss_dict
else:
offsets, cls_scores = prob[:, :-self.n], prob[:, -self.n:]
pred_bbox = offsets.reshape(-1, self.n, 4)
cls_prob = F.softmax(cls_scores, axis=1)
n = rcnn_rois.shape[0]
rois = F.broadcast_to(F.expand_dims(rcnn_rois[:, 1:5], axis=1), (n, 1, 4)).reshape(-1, 4)
normalized = config.rcnn_bbox_normalize_targets
pred_boxes = restore_bbox(rois, pred_bbox, normalized, config)
pred_bbox = F.concat([pred_boxes, F.expand_dims(cls_prob, axis=2)], axis=2)
return pred_bbox
def forward(self, features, im_info, boxes=None):
# prediction
pred_cls_score_list = []
pred_bbox_offsets_list = []
for x in features:
t = F.relu(self.rpn_conv(x))
pred_cls_score_list.append(self.rpn_cls_score(t))
pred_bbox_offsets_list.append(self.rpn_bbox_offsets(t))
# get anchors
all_anchors_list = []
fm_stride = 2 ** (len(features) + 1)
for fm in features:
layer_anchors = self.anchors_generator(fm, fm_stride)
fm_stride = fm_stride // 2
all_anchors_list.append(layer_anchors)
# sample from the predictions
rpn_rois, rpn_probs = find_top_rpn_proposals(
self.training, pred_bbox_offsets_list, pred_cls_score_list,
all_anchors_list, im_info)
if self.training:
rpn_labels, rpn_bbox_targets = fpn_anchor_target(
boxes, im_info, all_anchors_list)
#rpn_labels = rpn_labels.astype(np.int32)
pred_cls_score, pred_bbox_offsets = fpn_rpn_reshape(
pred_cls_score_list, pred_bbox_offsets_list)
# rpn loss
rpn_cls_loss = softmax_loss(pred_cls_score, rpn_labels)
rpn_bbox_loss = smooth_l1_loss_rpn(pred_bbox_offsets, rpn_bbox_targets, \
rpn_labels, config.rpn_smooth_l1_beta)
loss_dict = {}
loss_dict['loss_rpn_cls'] = rpn_cls_loss
loss_dict['loss_rpn_loc'] = rpn_bbox_loss
return rpn_rois, loss_dict
else:
return rpn_rois
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
pool_features, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align',
labels, bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for regression
labels = labels.astype(np.int32).reshape(-1)
pos_masks = labels > 0
localization_loss = smooth_l1_loss(
pred_delta,
bbox_targets,
config.rcnn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
# loss for classification
valid_masks = labels >= 0
objectness_loss = softmax_loss(
pred_cls,
labels)
objectness_loss = objectness_loss * valid_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rcnn_cls = objectness_loss.sum() * normalizer
loss_rcnn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict['loss_rcnn_cls'] = loss_rcnn_cls
loss_dict['loss_rcnn_loc'] = loss_rcnn_loc
return loss_dict
else:
pred_scores = F.softmax(pred_cls)
pred_bbox = restore_bbox(rcnn_rois[:, 1:5], pred_delta, True)
pred_bbox = F.concat([pred_bbox, pred_scores[:, 1].reshape(-1,1)], axis=1)
return pred_bbox
def emd_loss(p_b0, p_c0, p_b1, p_c1, targets, labels):
pred_box = F.concat([p_b0, p_b1], axis=1).reshape(-1, p_b0.shapeof()[-1])
pred_box = pred_box.reshape(-1, config.num_classes, 4)
pred_score = F.concat([p_c0, p_c1], axis=1).reshape(-1, p_c0.shapeof()[-1])
targets = targets.reshape(-1, 4)
labels = labels.reshape(-1).astype(np.int32)
fg_masks = F.greater(labels, 0)
non_ignore_masks = F.greater_equal(labels, 0)
# mulitple class to one
indexing_label = (labels * fg_masks).reshape(-1,1)
indexing_label = indexing_label.broadcast((labels.shapeof()[0], 4))
pred_box = F.indexing_one_hot(pred_box, indexing_label, 1)
# loss for regression
loss_box_reg = smooth_l1_loss(
pred_box,
targets,
config.rcnn_smooth_l1_beta)
# loss for classification
loss_cls = softmax_loss(pred_score, labels)
loss = loss_cls*non_ignore_masks + loss_box_reg * fg_masks
loss = loss.reshape(-1, 2).sum(axis=1)
return loss.reshape(-1, 1)
