def forward(self, pred_cls_list, rpn_num_prob_list, pred_reg_list, anchors_list, rpn_iou_list, boxes, im_info): all_anchors_list = [ F.concat([a, i * F.ones([a.shape[0], 1]).to(a.device)], axis=1) for i, a in enumerate(anchors_list) ] all_anchors_final = F.concat(all_anchors_list, axis=0) rpn_bbox_offset_final = F.concat(pred_reg_list, axis=1) rpn_cls_prob_final = F.concat(pred_cls_list, axis=1) rpn_iou_prob_final = F.concat(rpn_iou_list, axis=1) rpn_num_per_points_final = F.concat(rpn_num_prob_list, axis=1) rpn_labels, rpn_target_boxes = rpn_anchor_target_opr( boxes, im_info, all_anchors_final) ious_target = self.anchor_iou_target_opr(boxes, im_info, all_anchors_final, rpn_bbox_offset_final) n = rpn_labels.shape[0] target_boxes = rpn_target_boxes.reshape(n, -1, 4) rpn_cls_prob_final = rpn_cls_prob_final.reshape(n, -1, 1) offsets_final = rpn_bbox_offset_final.reshape(n, -1, 4) rpn_labels = rpn_labels.transpose(2, 0, 1) a, b = rpn_labels[0], rpn_labels[1] ignores = b - F.equal(a, 0).astype(np.float32) * F.equal(b, 0).astype( np.float32) labels = F.stack([a, ignores], axis=2).reshape(n, -1) cls_loss = sigmoid_cross_entropy_retina(rpn_cls_prob_final, labels, alpha=config.focal_loss_alpha, gamma=config.focal_loss_gamma) rpn_bbox_loss = smooth_l1_loss_retina(offsets_final, target_boxes, labels) rpn_labels = labels.reshape(n, -1, 2) rpn_iou_loss = iou_l1_loss(rpn_iou_prob_final, ious_target, rpn_labels) # whether one anchor produce one proposal or two. nlabels = ((labels.reshape(n, -1, 2) > 0).sum(2)).flatten() - 1 c = rpn_num_per_points_final.shape[2] num_per_anchor = rpn_num_per_points_final.reshape(-1, c) rpn_num_per_points_final = rpn_num_per_points_final.reshape(-1, c) nlabels = nlabels.reshape(-1) rpn_num_loss = softmax_loss(rpn_num_per_points_final, nlabels) loss_dict = {} loss_dict['rpn_cls_loss'] = cls_loss loss_dict['rpn_bbox_loss'] = 2 * rpn_bbox_loss loss_dict['rpn_iou_loss'] = 2 * rpn_iou_loss loss_dict['rpn_num_loss'] = rpn_num_loss return loss_dict
def forward(self, features, im_info, boxes=None): # prediction pred_cls_score_list = [] pred_bbox_offsets_list = [] for x in features: t = F.relu(self.rpn_conv(x)) pred_cls_score_list.append(self.rpn_cls_score(t)) pred_bbox_offsets_list.append(self.rpn_bbox_offsets(t)) # get anchors all_anchors_list = [] fm_stride = 2**(len(features) + 1) for fm in features: layer_anchors = self.anchors_generator(fm, fm_stride) fm_stride = fm_stride // 2 all_anchors_list.append(layer_anchors) # sample from the predictions rpn_rois, rpn_probs = find_top_rpn_proposals(self.training, pred_bbox_offsets_list, pred_cls_score_list, all_anchors_list, im_info) if self.training: rpn_labels, rpn_bbox_targets = fpn_anchor_target( boxes, im_info, all_anchors_list) #rpn_labels = rpn_labels.astype(np.int32) pred_cls_score, pred_bbox_offsets = fpn_rpn_reshape( pred_cls_score_list, pred_bbox_offsets_list) # rpn loss valid_masks = rpn_labels >= 0 valid_inds = mask_to_inds(valid_masks) objectness_loss = softmax_loss(pred_cls_score.ai[valid_inds], rpn_labels.ai[valid_inds]) #objectness_loss = objectness_loss * valid_masks pos_masks = rpn_labels > 0 localization_loss = smooth_l1_loss(pred_bbox_offsets, rpn_bbox_targets, config.rpn_smooth_l1_beta) localization_loss = localization_loss * pos_masks normalizer = 1.0 / (valid_masks.sum()) loss_rpn_cls = objectness_loss.sum() * normalizer loss_rpn_loc = localization_loss.sum() * normalizer loss_dict = {} loss_dict['loss_rpn_cls'] = loss_rpn_cls loss_dict['loss_rpn_loc'] = loss_rpn_loc return rpn_rois, loss_dict else: return rpn_rois
def emd_loss(p_b0, p_c0, p_b1, p_c1, targets, labels): pred_box = F.concat([p_b0, p_b1], axis=1).reshape(-1, p_b0.shapeof()[-1]) pred_score = F.concat([p_c0, p_c1], axis=1).reshape(-1, p_c0.shapeof()[-1]) targets = targets.reshape(-1, 4) labels = labels.reshape(-1) fg_masks = F.greater(labels, 0) non_ignore_masks = F.greater_equal(labels, 0) # loss for regression loss_box_reg = smooth_l1_loss(pred_box, targets, config.rcnn_smooth_l1_beta) # loss for classification loss_cls = softmax_loss(pred_score, labels) loss = loss_cls * non_ignore_masks + loss_box_reg * fg_masks loss = loss.reshape(-1, 2).sum(axis=1) return loss.reshape(-1, 1)
def compute_regular_loss(self, prob, bbox_targets, labels): offsets, cls_scores = prob[:, :-self.n], prob[:, -self.n:] n = offsets.shape[0] offsets = offsets.reshape(n, -1, 4) cls_loss = softmax_loss(cls_scores, labels) bbox_loss = smooth_l1_loss_rcnn_opr(offsets, bbox_targets, labels, config.rcnn_smooth_l1_beta) bbox_loss = bbox_loss.sum() / F.maximum((labels > 0).sum(), 1) loss = {} loss['{}_cls_loss'.format(self.name)] = cls_loss loss['{}_bbox_loss'.format(self.name)] = bbox_loss return loss
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None): # stride: 64,32,16,8,4 -> 4, 8, 16, 32 fpn_fms = fpn_fms[1:][::-1] stride = [4, 8, 16, 32] pool_features, rcnn_rois, labels, bbox_targets = roi_pool( fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align', labels, bbox_targets) flatten_feature = F.flatten(pool_features, start_axis=1) roi_feature = F.relu(self.fc1(flatten_feature)) roi_feature = F.relu(self.fc2(roi_feature)) pred_cls = self.pred_cls(roi_feature) pred_delta = self.pred_delta(roi_feature) if self.training: # loss for regression labels = labels.astype(np.int32).reshape(-1) # mulitple class to one pos_masks = labels > 0 pred_delta = pred_delta.reshape(-1, config.num_classes, 4) indexing_label = (labels * pos_masks).reshape(-1, 1) indexing_label = indexing_label.broadcast((labels.shapeof()[0], 4)) pred_delta = F.indexing_one_hot(pred_delta, indexing_label, 1) localization_loss = smooth_l1_loss(pred_delta, bbox_targets, config.rcnn_smooth_l1_beta) localization_loss = localization_loss * pos_masks # loss for classification valid_masks = labels >= 0 objectness_loss = softmax_loss(pred_cls, labels) objectness_loss = objectness_loss * valid_masks normalizer = 1.0 / (valid_masks.sum()) loss_rcnn_cls = objectness_loss.sum() * normalizer loss_rcnn_loc = localization_loss.sum() * normalizer loss_dict = {} loss_dict['loss_rcnn_cls'] = loss_rcnn_cls loss_dict['loss_rcnn_loc'] = loss_rcnn_loc return loss_dict else: pred_scores = F.softmax(pred_cls)[:, 1:].reshape(-1, 1) pred_delta = pred_delta[:, 4:].reshape(-1, 4) target_shape = (rcnn_rois.shapeof()[0], config.num_classes - 1, 4) base_rois = F.add_axis(rcnn_rois[:, 1:5], 1).broadcast(target_shape).reshape(-1, 4) pred_bbox = restore_bbox(base_rois, pred_delta, True) pred_bbox = F.concat([pred_bbox, pred_scores], axis=1) return pred_bbox
def forward(self, fpn_fms, proposals, labels=None, bbox_targets=None): # input p2-p5 fpn_fms = fpn_fms[1:][::-1] stride = [4, 8, 16, 32] #pool_features = roi_pooler(fpn_fms, proposals, stride, (7, 7), "ROIAlignV2") pool_features, proposals, labels, bbox_targets = roi_pool( fpn_fms, proposals, stride, (7, 7), 'roi_align', labels, bbox_targets) flatten_feature = F.flatten(pool_features, start_axis=1) roi_feature = F.relu(self.fc1(flatten_feature)) roi_feature = F.relu(self.fc2(roi_feature)) pred_cls = self.pred_cls(roi_feature) pred_delta = self.pred_delta(roi_feature) if self.training: # loss for regression labels = labels.astype(np.int32).reshape(-1) # mulitple class to one pos_masks = labels > 0 localization_loss = smooth_l1_loss( pred_delta, bbox_targets, config.rcnn_smooth_l1_beta) localization_loss = localization_loss * pos_masks # loss for classification valid_masks = labels >= 0 objectness_loss = softmax_loss( pred_cls, labels) objectness_loss = objectness_loss * valid_masks normalizer = 1.0 / (valid_masks.sum()) loss_rcnn_cls = objectness_loss.sum() * normalizer loss_rcnn_loc = localization_loss.sum() * normalizer loss_dict = {} loss_dict[self.stage_name + '_cls'] = loss_rcnn_cls loss_dict[self.stage_name + '_loc'] = loss_rcnn_loc pred_bbox = restore_bbox(proposals[:, 1:5], pred_delta, True) pred_proposals = F.zero_grad(F.concat([proposals[:, 0].reshape(-1, 1), pred_bbox], axis=1)) return pred_proposals, loss_dict else: pred_scores = F.softmax(pred_cls)[:, 1].reshape(-1, 1) pred_bbox = restore_bbox(proposals[:, 1:5], pred_delta, True) pred_proposals = F.concat([proposals[:, 0].reshape(-1, 1), pred_bbox], axis=1) return pred_proposals, pred_scores
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None): # stride: 64,32,16,8,4 -> 4, 8, 16, 32 fpn_fms = fpn_fms[1:] fpn_fms.reverse() stride = [4, 8, 16, 32] poo5, rcnn_rois, labels, bbox_targets = roi_pool( fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align', labels, bbox_targets) poo5 = F.flatten(poo5, start_axis=1) fc1 = F.relu(self.fc1(poo5)) fc2 = F.relu(self.fc2(fc1)) cls_scores = self.cls(fc2) pred_boxes = self.bbox(fc2) # a = self.a(fc2) # b = self.b(fc2) # prob = F.stack([a, b], axis=1).reshape(-1, a.shape[1]) prob = F.concat([pred_boxes, cls_scores], axis=1) if self.training: # emd_loss = self.compute_gemini_loss(prob, bbox_targets, labels) bbox_targets, labels = bbox_targets.reshape(-1, 4), labels.flatten() cls_loss = softmax_loss(cls_scores, labels) pred_boxes = pred_boxes.reshape(-1, self.n, 4) bbox_loss = smooth_l1_loss_rcnn(pred_boxes, bbox_targets, labels, \ config.rcnn_smooth_l1_beta) loss_dict = {} loss_dict['cls_loss'] = cls_loss loss_dict['bbox_loss'] = bbox_loss return loss_dict else: offsets, cls_scores = prob[:, :-self.n], prob[:, -self.n:] pred_bbox = offsets.reshape(-1, self.n, 4) cls_prob = F.softmax(cls_scores, axis=1) n = rcnn_rois.shape[0] rois = F.broadcast_to(F.expand_dims(rcnn_rois[:, 1:5], axis=1), (n, 1, 4)).reshape(-1, 4) normalized = config.rcnn_bbox_normalize_targets pred_boxes = restore_bbox(rois, pred_bbox, normalized, config) pred_bbox = F.concat([pred_boxes, F.expand_dims(cls_prob, axis=2)], axis=2) return pred_bbox
def forward(self, features, im_info, boxes=None): # prediction pred_cls_score_list = [] pred_bbox_offsets_list = [] for x in features: t = F.relu(self.rpn_conv(x)) pred_cls_score_list.append(self.rpn_cls_score(t)) pred_bbox_offsets_list.append(self.rpn_bbox_offsets(t)) # get anchors all_anchors_list = [] fm_stride = 2 ** (len(features) + 1) for fm in features: layer_anchors = self.anchors_generator(fm, fm_stride) fm_stride = fm_stride // 2 all_anchors_list.append(layer_anchors) # sample from the predictions rpn_rois, rpn_probs = find_top_rpn_proposals( self.training, pred_bbox_offsets_list, pred_cls_score_list, all_anchors_list, im_info) if self.training: rpn_labels, rpn_bbox_targets = fpn_anchor_target( boxes, im_info, all_anchors_list) #rpn_labels = rpn_labels.astype(np.int32) pred_cls_score, pred_bbox_offsets = fpn_rpn_reshape( pred_cls_score_list, pred_bbox_offsets_list) # rpn loss rpn_cls_loss = softmax_loss(pred_cls_score, rpn_labels) rpn_bbox_loss = smooth_l1_loss_rpn(pred_bbox_offsets, rpn_bbox_targets, \ rpn_labels, config.rpn_smooth_l1_beta) loss_dict = {} loss_dict['loss_rpn_cls'] = rpn_cls_loss loss_dict['loss_rpn_loc'] = rpn_bbox_loss return rpn_rois, loss_dict else: return rpn_rois
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None): # stride: 64,32,16,8,4 -> 4, 8, 16, 32 fpn_fms = fpn_fms[1:][::-1] stride = [4, 8, 16, 32] pool_features, rcnn_rois, labels, bbox_targets = roi_pool( fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align', labels, bbox_targets) flatten_feature = F.flatten(pool_features, start_axis=1) roi_feature = F.relu(self.fc1(flatten_feature)) roi_feature = F.relu(self.fc2(roi_feature)) pred_cls = self.pred_cls(roi_feature) pred_delta = self.pred_delta(roi_feature) if self.training: # loss for regression labels = labels.astype(np.int32).reshape(-1) pos_masks = labels > 0 localization_loss = smooth_l1_loss( pred_delta, bbox_targets, config.rcnn_smooth_l1_beta) localization_loss = localization_loss * pos_masks # loss for classification valid_masks = labels >= 0 objectness_loss = softmax_loss( pred_cls, labels) objectness_loss = objectness_loss * valid_masks normalizer = 1.0 / (valid_masks.sum()) loss_rcnn_cls = objectness_loss.sum() * normalizer loss_rcnn_loc = localization_loss.sum() * normalizer loss_dict = {} loss_dict['loss_rcnn_cls'] = loss_rcnn_cls loss_dict['loss_rcnn_loc'] = loss_rcnn_loc return loss_dict else: pred_scores = F.softmax(pred_cls) pred_bbox = restore_bbox(rcnn_rois[:, 1:5], pred_delta, True) pred_bbox = F.concat([pred_bbox, pred_scores[:, 1].reshape(-1,1)], axis=1) return pred_bbox
def emd_loss(p_b0, p_c0, p_b1, p_c1, targets, labels): pred_box = F.concat([p_b0, p_b1], axis=1).reshape(-1, p_b0.shapeof()[-1]) pred_box = pred_box.reshape(-1, config.num_classes, 4) pred_score = F.concat([p_c0, p_c1], axis=1).reshape(-1, p_c0.shapeof()[-1]) targets = targets.reshape(-1, 4) labels = labels.reshape(-1).astype(np.int32) fg_masks = F.greater(labels, 0) non_ignore_masks = F.greater_equal(labels, 0) # mulitple class to one indexing_label = (labels * fg_masks).reshape(-1,1) indexing_label = indexing_label.broadcast((labels.shapeof()[0], 4)) pred_box = F.indexing_one_hot(pred_box, indexing_label, 1) # loss for regression loss_box_reg = smooth_l1_loss( pred_box, targets, config.rcnn_smooth_l1_beta) # loss for classification loss_cls = softmax_loss(pred_score, labels) loss = loss_cls*non_ignore_masks + loss_box_reg * fg_masks loss = loss.reshape(-1, 2).sum(axis=1) return loss.reshape(-1, 1)