def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
pool_features, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align', labels,
bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for regression
labels = labels.astype(np.int32).reshape(-1)
# mulitple class to one
pos_masks = labels > 0
pred_delta = pred_delta.reshape(-1, config.num_classes, 4)
indexing_label = (labels * pos_masks).reshape(-1, 1)
indexing_label = indexing_label.broadcast((labels.shapeof()[0], 4))
pred_delta = F.indexing_one_hot(pred_delta, indexing_label, 1)
localization_loss = smooth_l1_loss(pred_delta, bbox_targets,
config.rcnn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
# loss for classification
valid_masks = labels >= 0
objectness_loss = softmax_loss(pred_cls, labels)
objectness_loss = objectness_loss * valid_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rcnn_cls = objectness_loss.sum() * normalizer
loss_rcnn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict['loss_rcnn_cls'] = loss_rcnn_cls
loss_dict['loss_rcnn_loc'] = loss_rcnn_loc
return loss_dict
else:
pred_scores = F.softmax(pred_cls)[:, 1:].reshape(-1, 1)
pred_delta = pred_delta[:, 4:].reshape(-1, 4)
target_shape = (rcnn_rois.shapeof()[0], config.num_classes - 1, 4)
base_rois = F.add_axis(rcnn_rois[:, 1:5],
1).broadcast(target_shape).reshape(-1, 4)
pred_bbox = restore_bbox(base_rois, pred_delta, True)
pred_bbox = F.concat([pred_bbox, pred_scores], axis=1)
return pred_bbox
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
pool_features, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align',
labels, bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_emd_pred_cls_0 = self.emd_pred_cls_0(roi_feature)
pred_emd_pred_delta_0 = self.emd_pred_delta_0(roi_feature)
pred_emd_pred_cls_1 = self.emd_pred_cls_1(roi_feature)
pred_emd_pred_delta_1 = self.emd_pred_delta_1(roi_feature)
if self.training:
loss0 = emd_loss(
pred_emd_pred_delta_0, pred_emd_pred_cls_0,
pred_emd_pred_delta_1, pred_emd_pred_cls_1,
bbox_targets, labels)
loss1 = emd_loss(
pred_emd_pred_delta_1, pred_emd_pred_cls_1,
pred_emd_pred_delta_0, pred_emd_pred_cls_0,
bbox_targets, labels)
loss = F.concat([loss0, loss1], axis=1)
indices = F.argmin(loss, axis=1)
loss_emd = F.indexing_one_hot(loss, indices, 1)
loss_emd = loss_emd.sum()/loss_emd.shapeof()[0]
loss_dict = {}
loss_dict['loss_rcnn_emd'] = loss_emd
return loss_dict
else:
pred_scores_0 = F.softmax(pred_emd_pred_cls_0)[:, 1:].reshape(-1, 1)
pred_scores_1 = F.softmax(pred_emd_pred_cls_1)[:, 1:].reshape(-1, 1)
pred_delta_0 = pred_emd_pred_delta_0[:, 4:].reshape(-1, 4)
pred_delta_1 = pred_emd_pred_delta_1[:, 4:].reshape(-1, 4)
target_shape = (rcnn_rois.shapeof()[0], config.num_classes - 1, 4)
base_rois = F.add_axis(rcnn_rois[:, 1:5], 1).broadcast(target_shape).reshape(-1, 4)
pred_bbox_0 = restore_bbox(base_rois, pred_delta_0, True)
pred_bbox_1 = restore_bbox(base_rois, pred_delta_1, True)
pred_bbox_0 = F.concat([pred_bbox_0, pred_scores_0], axis=1)
pred_bbox_1 = F.concat([pred_bbox_1, pred_scores_1], axis=1)
#[{head0, pre1, tag1}, {head1, pre1, tag1}, {head0, pre1, tag2}, ...]
pred_bbox = F.concat((pred_bbox_0, pred_bbox_1), axis=1).reshape(-1,5)
return pred_bbox
def forward(self, fpn_fms, proposals, labels=None, bbox_targets=None):
# input p2-p5
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
#pool_features = roi_pooler(fpn_fms, proposals, stride, (7, 7), "ROIAlignV2")
pool_features, proposals, labels, bbox_targets = roi_pool(
fpn_fms, proposals, stride, (7, 7), 'roi_align',
labels, bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for regression
labels = labels.astype(np.int32).reshape(-1)
# mulitple class to one
pos_masks = labels > 0
localization_loss = smooth_l1_loss(
pred_delta,
bbox_targets,
config.rcnn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
# loss for classification
valid_masks = labels >= 0
objectness_loss = softmax_loss(
pred_cls,
labels)
objectness_loss = objectness_loss * valid_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rcnn_cls = objectness_loss.sum() * normalizer
loss_rcnn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict[self.stage_name + '_cls'] = loss_rcnn_cls
loss_dict[self.stage_name + '_loc'] = loss_rcnn_loc
pred_bbox = restore_bbox(proposals[:, 1:5], pred_delta, True)
pred_proposals = F.zero_grad(F.concat([proposals[:, 0].reshape(-1, 1), pred_bbox], axis=1))
return pred_proposals, loss_dict
else:
pred_scores = F.softmax(pred_cls)[:, 1].reshape(-1, 1)
pred_bbox = restore_bbox(proposals[:, 1:5], pred_delta, True)
pred_proposals = F.concat([proposals[:, 0].reshape(-1, 1), pred_bbox], axis=1)
return pred_proposals, pred_scores
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:]
fpn_fms.reverse()
stride = [4, 8, 16, 32]
poo5, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align',
labels, bbox_targets)
poo5 = F.flatten(poo5, start_axis=1)
fc1 = F.relu(self.fc1(poo5))
fc2 = F.relu(self.fc2(fc1))
cls_scores = self.cls(fc2)
pred_boxes = self.bbox(fc2)
# a = self.a(fc2)
# b = self.b(fc2)
# prob = F.stack([a, b], axis=1).reshape(-1, a.shape[1])
prob = F.concat([pred_boxes, cls_scores], axis=1)
if self.training:
# emd_loss = self.compute_gemini_loss(prob, bbox_targets, labels)
bbox_targets, labels = bbox_targets.reshape(-1, 4), labels.flatten()
cls_loss = softmax_loss(cls_scores, labels)
pred_boxes = pred_boxes.reshape(-1, self.n, 4)
bbox_loss = smooth_l1_loss_rcnn(pred_boxes, bbox_targets, labels, \
config.rcnn_smooth_l1_beta)
loss_dict = {}
loss_dict['cls_loss'] = cls_loss
loss_dict['bbox_loss'] = bbox_loss
return loss_dict
else:
offsets, cls_scores = prob[:, :-self.n], prob[:, -self.n:]
pred_bbox = offsets.reshape(-1, self.n, 4)
cls_prob = F.softmax(cls_scores, axis=1)
n = rcnn_rois.shape[0]
rois = F.broadcast_to(F.expand_dims(rcnn_rois[:, 1:5], axis=1), (n, 1, 4)).reshape(-1, 4)
normalized = config.rcnn_bbox_normalize_targets
pred_boxes = restore_bbox(rois, pred_bbox, normalized, config)
pred_bbox = F.concat([pred_boxes, F.expand_dims(cls_prob, axis=2)], axis=2)
return pred_bbox
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:]
fpn_fms.reverse()
stride = [4, 8, 16, 32]
poo5, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align', labels,
bbox_targets)
poo5 = F.flatten(poo5, start_axis=1)
fc1 = F.relu(self.fc1(poo5))
fc2 = F.relu(self.fc2(fc1))
a = self.a(fc2)
b = self.b(fc2)
prob = F.stack([a, b], axis=1).reshape(-1, a.shape[1])
if self.refinement:
final_prob = self.refinement_module(prob, fc2)
if self.training:
emd_loss = self.compute_gemini_loss(prob, bbox_targets, labels)
loss_dict = {}
loss_dict['loss_rcnn_emd'] = emd_loss
if self.refinement_module:
final_emd_loss = self.compute_gemini_loss(
final_prob, bbox_targets, labels)
loss_dict['final_rcnn_emd'] = final_emd_loss
return loss_dict
else:
offsets, cls_scores = prob[:, :-self.n], prob[:, -self.n:]
pred_bbox = offsets.reshape(-1, self.n, 4)
cls_prob = F.softmax(cls_scores, axis=1)
n = rcnn_rois.shape[0]
rois = F.broadcast_to(F.expand_dims(rcnn_rois[:, 1:5], axis=1),
(n, 2, 4)).reshape(-1, 4)
normalized = config.rcnn_bbox_normalize_targets
pred_boxes = restore_bbox(rois, pred_bbox, normalized, config)
pred_bbox = F.concat(
[pred_boxes, F.expand_dims(cls_prob, axis=2)], axis=2)
return pred_bbox
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
pool_features, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align',
labels, bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_cls = self.pred_cls(roi_feature)
pred_delta = self.pred_delta(roi_feature)
if self.training:
# loss for regression
labels = labels.astype(np.int32).reshape(-1)
pos_masks = labels > 0
localization_loss = smooth_l1_loss(
pred_delta,
bbox_targets,
config.rcnn_smooth_l1_beta)
localization_loss = localization_loss * pos_masks
# loss for classification
valid_masks = labels >= 0
objectness_loss = softmax_loss(
pred_cls,
labels)
objectness_loss = objectness_loss * valid_masks
normalizer = 1.0 / (valid_masks.sum())
loss_rcnn_cls = objectness_loss.sum() * normalizer
loss_rcnn_loc = localization_loss.sum() * normalizer
loss_dict = {}
loss_dict['loss_rcnn_cls'] = loss_rcnn_cls
loss_dict['loss_rcnn_loc'] = loss_rcnn_loc
return loss_dict
else:
pred_scores = F.softmax(pred_cls)
pred_bbox = restore_bbox(rcnn_rois[:, 1:5], pred_delta, True)
pred_bbox = F.concat([pred_bbox, pred_scores[:, 1].reshape(-1,1)], axis=1)
return pred_bbox
def forward(self, fpn_fms, rcnn_rois, labels=None, bbox_targets=None):
# stride: 64,32,16,8,4 -> 4, 8, 16, 32
fpn_fms = fpn_fms[1:][::-1]
stride = [4, 8, 16, 32]
pool_features, rcnn_rois, labels, bbox_targets = roi_pool(
fpn_fms, rcnn_rois, stride, (7, 7), 'roi_align',
labels, bbox_targets)
flatten_feature = F.flatten(pool_features, start_axis=1)
roi_feature = F.relu(self.fc1(flatten_feature))
roi_feature = F.relu(self.fc2(roi_feature))
pred_emd_pred_cls_0 = self.emd_pred_cls_0(roi_feature)
pred_emd_pred_delta_0 = self.emd_pred_delta_0(roi_feature)
pred_emd_pred_cls_1 = self.emd_pred_cls_1(roi_feature)
pred_emd_pred_delta_1 = self.emd_pred_delta_1(roi_feature)
pred_emd_scores_0 = F.softmax(pred_emd_pred_cls_0)
pred_emd_scores_1 = F.softmax(pred_emd_pred_cls_1)
# make refine feature
box_0 = F.concat((pred_emd_pred_delta_0,
pred_emd_scores_0[:, 1][:, None]), axis=1)[:, None, :]
box_1 = F.concat((pred_emd_pred_delta_1,
pred_emd_scores_1[:, 1][:, None]), axis=1)[:, None, :]
boxes_feature_0 = box_0.broadcast(
box_0.shapeof()[0], 4, box_0.shapeof()[-1]).reshape(box_0.shapeof()[0], -1)
boxes_feature_1 = box_1.broadcast(
box_1.shapeof()[0], 4, box_1.shapeof()[-1]).reshape(box_1.shapeof()[0], -1)
boxes_feature_0 = F.concat((roi_feature, boxes_feature_0), axis=1)
boxes_feature_1 = F.concat((roi_feature, boxes_feature_1), axis=1)
refine_feature_0 = F.relu(self.fc3(boxes_feature_0))
refine_feature_1 = F.relu(self.fc3(boxes_feature_1))
# refine
pred_ref_pred_cls_0 = self.ref_pred_cls_0(refine_feature_0)
pred_ref_pred_delta_0 = self.ref_pred_delta_0(refine_feature_0)
pred_ref_pred_cls_1 = self.ref_pred_cls_1(refine_feature_1)
pred_ref_pred_delta_1 = self.ref_pred_delta_1(refine_feature_1)
if self.training:
loss0 = emd_loss(
pred_emd_pred_delta_0, pred_emd_pred_cls_0,
pred_emd_pred_delta_1, pred_emd_pred_cls_1,
bbox_targets, labels)
loss1 = emd_loss(
pred_emd_pred_delta_1, pred_emd_pred_cls_1,
pred_emd_pred_delta_0, pred_emd_pred_cls_0,
bbox_targets, labels)
loss2 = emd_loss(
pred_ref_pred_delta_0, pred_ref_pred_cls_0,
pred_ref_pred_delta_1, pred_ref_pred_cls_1,
bbox_targets, labels)
loss3 = emd_loss(
pred_ref_pred_delta_1, pred_ref_pred_cls_1,
pred_ref_pred_delta_0, pred_ref_pred_cls_0,
bbox_targets, labels)
loss_rcnn = F.concat([loss0, loss1], axis=1)
loss_ref = F.concat([loss2, loss3], axis=1)
indices_rcnn = F.argmin(loss_rcnn, axis=1)
indices_ref = F.argmin(loss_ref, axis=1)
loss_rcnn = F.indexing_one_hot(loss_rcnn, indices_rcnn, 1)
loss_ref = F.indexing_one_hot(loss_ref, indices_ref, 1)
loss_rcnn = loss_rcnn.sum()/loss_rcnn.shapeof()[0]
loss_ref = loss_ref.sum()/loss_ref.shapeof()[0]
loss_dict = {}
loss_dict['loss_rcnn_emd'] = loss_rcnn
loss_dict['loss_ref_emd'] = loss_ref
return loss_dict
else:
pred_ref_scores_0 = F.softmax(pred_ref_pred_cls_0)
pred_ref_scores_1 = F.softmax(pred_ref_pred_cls_1)
pred_bbox_0 = restore_bbox(rcnn_rois[:, 1:5], pred_ref_pred_delta_0, True)
pred_bbox_1 = restore_bbox(rcnn_rois[:, 1:5], pred_ref_pred_delta_1, True)
pred_bbox_0 = F.concat([pred_bbox_0, pred_ref_scores_0[:, 1].reshape(-1,1)], axis=1)
pred_bbox_1 = F.concat([pred_bbox_1, pred_ref_scores_1[:, 1].reshape(-1,1)], axis=1)
pred_bbox = F.concat((pred_bbox_0, pred_bbox_1), axis=1).reshape(-1,5)
return pred_bbox