def forward(self, inputs):
image = self.preprocess_image(inputs["image"])
features = self.backbone(image)
features = [features[f] for f in self.in_features]
box_cls, box_delta = self.head(features)
box_cls_list = [
_.dimshuffle(0, 2, 3, 1).reshape(self.batch_size, -1, self.cfg.num_classes)
for _ in box_cls
]
box_delta_list = [
_.dimshuffle(0, 2, 3, 1).reshape(self.batch_size, -1, 4) for _ in box_delta
]
anchors_list = [
self.anchor_gen(features[i], self.stride_list[i]) for i in range(5)
]
all_level_box_cls = F.sigmoid(F.concat(box_cls_list, axis=1))
all_level_box_delta = F.concat(box_delta_list, axis=1)
all_level_anchors = F.concat(anchors_list, axis=0)
if self.training:
box_gt_cls, box_gt_delta = self.get_ground_truth(
all_level_anchors,
inputs["gt_boxes"],
inputs["im_info"][:, 4].astype(np.int32),
)
rpn_cls_loss = layers.get_focal_loss(
all_level_box_cls,
box_gt_cls,
alpha=self.cfg.focal_loss_alpha,
gamma=self.cfg.focal_loss_gamma,
)
rpn_bbox_loss = (
layers.get_smooth_l1_loss(all_level_box_delta, box_gt_delta, box_gt_cls)
* self.cfg.reg_loss_weight
)
total = rpn_cls_loss + rpn_bbox_loss
return total, rpn_cls_loss, rpn_bbox_loss
else:
# currently not support multi-batch testing
assert self.batch_size == 1
transformed_box = self.box_coder.decode(
all_level_anchors, all_level_box_delta[0],
)
transformed_box = transformed_box.reshape(-1, 4)
scale_w = inputs["im_info"][0, 1] / inputs["im_info"][0, 3]
scale_h = inputs["im_info"][0, 0] / inputs["im_info"][0, 2]
transformed_box = transformed_box / F.concat(
[scale_w, scale_h, scale_w, scale_h], axis=0
)
clipped_box = layers.get_clipped_box(
transformed_box, inputs["im_info"][0, 2:4]
).reshape(-1, 4)
return all_level_box_cls[0], clipped_box
def forward(self, features, im_info, boxes=None):
# prediction
features = [features[x] for x in self.in_features]
# get anchors
all_anchors_list = [
self.anchors_generator(fm, stride)
for fm, stride in zip(features, self.stride_list)
]
pred_cls_logit_list = []
pred_bbox_offset_list = []
for x in features:
t = F.relu(self.rpn_conv(x))
scores = self.rpn_cls_score(t)
pred_cls_logit_list.append(
scores.reshape(
scores.shape[0],
2,
self.num_cell_anchors,
scores.shape[2],
scores.shape[3],
))
bbox_offsets = self.rpn_bbox_offsets(t)
pred_bbox_offset_list.append(
bbox_offsets.reshape(
bbox_offsets.shape[0],
self.num_cell_anchors,
4,
bbox_offsets.shape[2],
bbox_offsets.shape[3],
))
# sample from the predictions
rpn_rois = self.find_top_rpn_proposals(pred_bbox_offset_list,
pred_cls_logit_list,
all_anchors_list, im_info)
if self.training:
rpn_labels, rpn_bbox_targets = self.get_ground_truth(
boxes, im_info, all_anchors_list)
pred_cls_logits, pred_bbox_offsets = self.merge_rpn_score_box(
pred_cls_logit_list, pred_bbox_offset_list)
# rpn loss
loss_rpn_cls = layers.softmax_loss(pred_cls_logits, rpn_labels)
loss_rpn_loc = layers.get_smooth_l1_loss(
pred_bbox_offsets,
rpn_bbox_targets,
rpn_labels,
self.cfg.rpn_smooth_l1_beta,
norm_type="all",
)
loss_dict = {
"loss_rpn_cls": loss_rpn_cls,
"loss_rpn_loc": loss_rpn_loc
}
return rpn_rois, loss_dict
else:
return rpn_rois
def forward(self, fpn_fms, rcnn_rois, im_info=None, gt_boxes=None):
rcnn_rois, labels, bbox_targets = self.get_ground_truth(
rcnn_rois, im_info, gt_boxes)
fpn_fms = [fpn_fms[x] for x in self.in_features]
pool_features = layers.roi_pool(
fpn_fms,
rcnn_rois,
self.stride,
self.pooling_size,
self.pooling_method,
)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for classification
loss_rcnn_cls = layers.softmax_loss(pred_cls, labels)
# loss for regression
pred_delta = pred_delta.reshape(-1, self.cfg.num_classes + 1, 4)
vlabels = labels.reshape(-1, 1).broadcast((labels.shapeof(0), 4))
pred_delta = F.indexing_one_hot(pred_delta, vlabels, axis=1)
loss_rcnn_loc = layers.get_smooth_l1_loss(
pred_delta,
bbox_targets,
labels,
self.cfg.rcnn_smooth_l1_beta,
norm_type="all",
)
loss_dict = {
'loss_rcnn_cls': loss_rcnn_cls,
'loss_rcnn_loc': loss_rcnn_loc
}
return loss_dict
else:
# slice 1 for removing background
pred_scores = F.softmax(pred_cls, axis=1)[:, 1:]
pred_delta = pred_delta[:, 4:].reshape(-1, 4)
target_shape = (rcnn_rois.shapeof(0), self.cfg.num_classes, 4)
# rois (N, 4) -> (N, 1, 4) -> (N, 80, 4) -> (N * 80, 4)
base_rois = F.add_axis(rcnn_rois[:, 1:5],
1).broadcast(target_shape).reshape(-1, 4)
pred_bbox = self.box_coder.decode(base_rois, pred_delta)
return pred_bbox, pred_scores
def forward(self, inputs):
image = self.preprocess_image(inputs["image"])
features = self.backbone(image)
features = [features[f] for f in self.in_features]
box_logits, box_offsets = self.head(features)
box_logits_list = [
_.dimshuffle(0, 2, 3, 1).reshape(self.batch_size, -1,
self.cfg.num_classes)
for _ in box_logits
]
box_offsets_list = [
_.dimshuffle(0, 2, 3, 1).reshape(self.batch_size, -1, 4)
for _ in box_offsets
]
anchors_list = [
self.anchor_gen(features[i], self.stride_list[i])
for i in range(len(features))
]
all_level_box_logits = F.concat(box_logits_list, axis=1)
all_level_box_offsets = F.concat(box_offsets_list, axis=1)
all_level_anchors = F.concat(anchors_list, axis=0)
if self.training:
box_gt_scores, box_gt_offsets = self.get_ground_truth(
all_level_anchors,
inputs["gt_boxes"],
inputs["im_info"][:, 4].astype(np.int32),
)
norm_type = "none" if self.cfg.loss_normalizer_momentum > 0.0 else "fg"
rpn_cls_loss = layers.get_focal_loss(
all_level_box_logits,
box_gt_scores,
alpha=self.cfg.focal_loss_alpha,
gamma=self.cfg.focal_loss_gamma,
norm_type=norm_type,
)
rpn_bbox_loss = (layers.get_smooth_l1_loss(
all_level_box_offsets,
box_gt_offsets,
box_gt_scores,
self.cfg.smooth_l1_beta,
norm_type=norm_type,
) * self.cfg.reg_loss_weight)
if norm_type == "none":
F.add_update(
self.loss_normalizer,
(box_gt_scores > 0).sum(),
alpha=self.cfg.loss_normalizer_momentum,
beta=1 - self.cfg.loss_normalizer_momentum,
)
rpn_cls_loss = rpn_cls_loss / F.maximum(
self.loss_normalizer, 1)
rpn_bbox_loss = rpn_bbox_loss / F.maximum(
self.loss_normalizer, 1)
total = rpn_cls_loss + rpn_bbox_loss
loss_dict = {
"total_loss": total,
"loss_cls": rpn_cls_loss,
"loss_loc": rpn_bbox_loss,
}
self.cfg.losses_keys = list(loss_dict.keys())
return loss_dict
else:
# currently not support multi-batch testing
assert self.batch_size == 1
transformed_box = self.box_coder.decode(
all_level_anchors,
all_level_box_offsets[0],
)
transformed_box = transformed_box.reshape(-1, 4)
scale_w = inputs["im_info"][0, 1] / inputs["im_info"][0, 3]
scale_h = inputs["im_info"][0, 0] / inputs["im_info"][0, 2]
transformed_box = transformed_box / F.concat(
[scale_w, scale_h, scale_w, scale_h], axis=0)
clipped_box = layers.get_clipped_box(
transformed_box, inputs["im_info"][0, 2:4]).reshape(-1, 4)
all_level_box_scores = F.sigmoid(all_level_box_logits)
return all_level_box_scores[0], clipped_box