def loss(self,
match_score,
ids,
id_weights,
reduce=True):
losses = dict()
if self.dynamic:
n = len(match_score)
x_n = [s.size(0) for s in match_score]
ids = torch.split(ids, x_n, dim=0)
loss_match = 0.
match_acc = 0.
n_total = 0
batch_size = len(ids)
for score, cur_ids, cur_weights in zip(match_score, ids, id_weights):
valid_idx = torch.nonzero(cur_weights).squeeze()
if len(valid_idx.size()) == 0: continue
n_valid = valid_idx.size(0)
n_total += n_valid
loss_match += weighted_cross_entropy(
score, cur_ids, cur_weights, reduce=reduce)
match_acc += accuracy(torch.index_select(score, 0, valid_idx),
torch.index_select(cur_ids, 0, valid_idx)) * n_valid
losses['loss_match'] = loss_match / n
if n_total > 0:
losses['match_acc'] = match_acc / n_total
else:
if match_score is not None:
valid_idx = torch.nonzero(cur_weights).squeeze()
losses['loss_match'] = weighted_cross_entropy(
match_score, ids, id_weights, reduce=reduce)
losses['match_acc'] = accuracy(torch.index_select(match_score, 0, valid_idx),
torch.index_select(ids, 0, valid_idx))
return losses
def loss(self,
cls_score,
bbox_pred,
labels,
label_weights,
bbox_targets,
bbox_weights,
reduce=True,
img_meta_2=None,
img_meta_3=None):
losses = dict()
if cls_score is not None:
if img_meta_3 is not None:
losses['loss_cls_3'] = weighted_cross_entropy(cls_score,
labels,
label_weights,
reduce=reduce)
losses['acc_3'] = accuracy(cls_score, labels)
elif img_meta_2 is not None:
losses['loss_cls_2'] = weighted_cross_entropy(cls_score,
labels,
label_weights,
reduce=reduce)
losses['acc_2'] = accuracy(cls_score, labels)
else:
losses['loss_cls'] = weighted_cross_entropy(cls_score,
labels,
label_weights,
reduce=reduce)
losses['acc'] = accuracy(cls_score, labels)
if bbox_pred is not None:
pos_inds = labels > 0
if self.reg_class_agnostic:
pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), 6)[pos_inds]
else:
pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), -1,
6)[pos_inds, labels[pos_inds]]
if img_meta_3 is not None:
losses['loss_reg_3'] = weighted_smoothl1(
pos_bbox_pred,
bbox_targets[pos_inds],
bbox_weights[pos_inds],
avg_factor=bbox_targets.size(0))
elif img_meta_2 is not None:
losses['loss_reg_2'] = weighted_smoothl1(
pos_bbox_pred,
bbox_targets[pos_inds],
bbox_weights[pos_inds],
avg_factor=bbox_targets.size(0))
else:
losses['loss_reg'] = weighted_smoothl1(
pos_bbox_pred,
bbox_targets[pos_inds],
bbox_weights[pos_inds],
avg_factor=bbox_targets.size(0))
return losses
def loss(
self,
cls_score,
bbox_pred,
labels,
label_weights,
bbox_targets,
bbox_weights,
reduce=True,
):
losses = dict()
if cls_score is not None:
losses["loss_cls"] = weighted_cross_entropy(cls_score,
labels,
label_weights,
reduce=reduce)
losses["acc"] = accuracy(cls_score, labels)
if bbox_pred is not None:
pos_inds = labels > 0
if self.reg_class_agnostic:
pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), 4)[pos_inds]
else:
pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), -1,
4)[pos_inds, labels[pos_inds]]
losses["loss_reg"] = weighted_smoothl1(
pos_bbox_pred,
bbox_targets[pos_inds],
bbox_weights[pos_inds],
avg_factor=bbox_targets.size(0),
)
return losses
def loss(self,
cls_score,
bbox_pred,
labels,
label_weights,
bbox_targets,
bbox_weights,
mix_inds=None,
reduce=True):
losses = dict()
if cls_score is not None:
losses['loss_cls'] = weighted_cross_entropy(cls_score,
labels,
label_weights,
mix_inds=mix_inds,
reduce=reduce)
losses['acc'] = accuracy(cls_score, labels)
if bbox_pred is not None:
losses['loss_reg'] = weighted_smoothl1(
bbox_pred,
bbox_targets,
bbox_weights,
mix_inds=mix_inds,
avg_factor=bbox_targets.size(0))
return losses
def loss_single(self, rpn_cls_score, rpn_bbox_pred, rpn_dir_pred, labels,
label_weights, bbox_targets, bbox_weights, dir_labels,
dir_weights, num_total_samples, cfg):
# classification loss
labels = labels.contiguous().view(-1)
label_weights = label_weights.contiguous().view(-1)
if self.use_sigmoid_cls:
rpn_cls_score = rpn_cls_score.permute(0, 2, 3,
1).contiguous().view(-1)
#criterion = weighted_binary_cross_entropy
criterion = weighted_sigmoid_focal_loss
else:
rpn_cls_score = rpn_cls_score.permute(0, 2, 3,
1).contiguous().view(-1, 2)
criterion = weighted_cross_entropy
loss_cls = criterion(rpn_cls_score,
labels,
label_weights,
avg_factor=num_total_samples)
# regression loss
bbox_targets = bbox_targets.contiguous().view(-1, 7)
bbox_weights = bbox_weights.contiguous().view(-1, 7)
rpn_bbox_pred = rpn_bbox_pred.permute(0, 2, 3,
1).contiguous().view(-1, 7)
rpn_bbox_pred, bbox_targets = add_sin_difference(
rpn_bbox_pred, bbox_targets)
loss_reg = weighted_smoothl1(rpn_bbox_pred,
bbox_targets,
bbox_weights,
beta=cfg.smoothl1_beta,
avg_factor=num_total_samples)
# direction loss
dir_logits = rpn_dir_pred.permute(0, 2, 3, 1).contiguous().view(-1, 2)
loss_dir = weighted_cross_entropy(dir_logits,
dir_labels,
dir_weights,
avg_factor=num_total_samples)
loss_reg *= 2
loss_dir *= .2
return loss_cls, loss_reg, loss_dir
def loss(self,
cls_score,
bbox_pred,
labels,
label_weights,
bbox_targets,
bbox_weights,
reduce=True):
losses = dict()
if cls_score is not None:
losses['loss_cls'] = weighted_cross_entropy(cls_score,
labels,
label_weights,
reduce=reduce)
losses['acc'] = accuracy(cls_score, labels)
return losses
def loss(self, cls_score, bbox_pred, A_pred, A_gt, labels, label_weights, bbox_targets,
bbox_weights, reduce=True):
losses = dict()
if cls_score is not None:
losses['loss_cls'] = weighted_cross_entropy(
cls_score, labels, label_weights, reduce=reduce)
losses['acc'] = accuracy(cls_score, labels)
if bbox_pred is not None:
losses['loss_reg'] = weighted_smoothl1(
bbox_pred,
bbox_targets,
bbox_weights,
avg_factor=bbox_targets.size(0))
if A_pred:
assert len(A_pred) == len(A_gt)
assert A_pred[0].size() == A_gt[0].size()
num_a_pred = len(A_pred)
for i in range(num_a_pred):
losses['loss_adj' + str(i)] = F.mse_loss(A_pred[i], A_gt[i].detach())
return losses
def loss(self,
cls_score,
bbox_pred,
labels,
label_weights,
bbox_targets,
bbox_weights,
alpha=0.25,
num_level=4,
reduce=True):
losses = dict()
for i in range(num_level):
cls_score_level_i = cls_score[i::num_level, :]
bbox_pred_level_i = bbox_pred[i::num_level, :]
losses['loss_cls_level%d' % i] = weighted_cross_entropy(
cls_score_level_i, labels, label_weights,
reduce=reduce) * alpha
losses['loss_reg_level%d' % i] = weighted_smoothl1(
bbox_pred_level_i,
bbox_targets,
bbox_weights,
avg_factor=bbox_targets.size(0)) * alpha
return losses
def forward_train(self,
img,
img_meta,
gt_bboxes,
gt_bboxes_8_coo,
gt_labels,
gt_bboxes_ignore=None,
gt_masks=None,
proposals=None):
if gt_bboxes_ignore is None:
gt_bboxes_ignore = [None for _ in range(len(gt_bboxes))]
x = self.extract_feat(img)
losses = dict()
# RPN forward and loss
if self.with_rpn:
rpn_outs = self.rpn_head(x)
rpn_loss_inputs = rpn_outs + (gt_bboxes, gt_bboxes_8_coo, img_meta, self.coo_num,
self.train_cfg.rpn)
rpn_losses = self.rpn_head.loss(*rpn_loss_inputs)
losses.update(rpn_losses)
proposal_inputs = rpn_outs + (img_meta, self.coo_num, self.test_cfg.rpn)
proposal_list = self.rpn_head.get_proposals(*proposal_inputs)
else:
proposal_list = proposals
if self.train_cfg.rpn.with_gt_bboxes:
for i in range(len(gt_bboxes)):
gt_bboxes_trans = []
for j in self.train_cfg.rpn.gt_bboxes_scale:
x_center = (gt_bboxes[i][:, 0:1]+gt_bboxes[i][:, 2:3])/2
y_center = (gt_bboxes[i][:, 1:2]+gt_bboxes[i][:, 3:4])/2
left = torch.clamp(((gt_bboxes[i][:, 0:1] - x_center) * j + x_center), min=0)
right = torch.clamp(((gt_bboxes[i][:, 2:3] - x_center) * j + x_center), max=img_meta[i]['img_shape'][1])
top = torch.clamp(((gt_bboxes[i][:, 1:2] - y_center) * j + y_center), min=0)
bottom = torch.clamp(((gt_bboxes[i][:, 3:4] - y_center) * j + y_center), max=img_meta[i]['img_shape'][0])
trans_gt_bboxes = torch.cat([left, top, right, bottom], 1)
gt_bboxes_trans.append(trans_gt_bboxes)
gt_bboxes_trans = torch.cat(gt_bboxes_trans, 0)
n = gt_bboxes_trans.shape[0]
gt_bboxes_trans = torch.cat([gt_bboxes_trans, torch.ones([n, 1], device=gt_bboxes[i].device)], 1)
proposal_list[i] = torch.cat([proposal_list[i], gt_bboxes_trans], 0)
# assign gts and sample proposals
_gt_bboxes_8_coo = [None for i in range(len(proposal_list))]
if self.with_bbox or self.with_mask:
assign_results, sampling_results = multi_apply(
assign_and_sample,
proposal_list,
gt_bboxes,
_gt_bboxes_8_coo,
gt_bboxes_ignore,
gt_labels,
cfg=self.train_cfg.rcnn)
# bbox head forward and loss
if self.with_bbox:
rois = bbox2roi([res.bboxes for res in sampling_results])
# TODO: a more flexible way to decide which feature maps to use
bbox_feats = self.bbox_roi_extractor(
x[:self.bbox_roi_extractor.num_inputs], rois, self.level, self.merge_mode)
cls_score, bbox_pred = self.bbox_head(bbox_feats)
bbox_targets = self.bbox_head.get_target(
sampling_results, gt_bboxes, gt_labels, self.train_cfg.rcnn)
loss_bbox = self.bbox_head.loss(cls_score, bbox_pred,
*bbox_targets)
if self.train_cfg.rcnn.with_hard_example_mining:
pred_label = torch.argmax(cls_score, 1)
labels, label_weights, bbox_gt, bbox_weights = bbox_targets
ind = pred_label != labels
if torch.sum(ind).item() != 0:
x_stop_grad = [feature.data for feature in x]
bbox_feats_stop_grad = self.bbox_roi_extractor(
x_stop_grad[:self.bbox_roi_extractor.num_inputs], rois, self.level, self.merge_mode)
cls_score_stop_grad, bbox_pred_stop_grad = self.bbox_head(bbox_feats_stop_grad)
cls_score_stop_grad, labels, label_weights = cls_score_stop_grad[ind], labels[ind], label_weights[ind]
num = cls_score.shape[0]
loss_bbox['loss_cls'] = loss_bbox['loss_cls'] + weighted_cross_entropy(
cls_score_stop_grad, labels, label_weights, avg_factor=num)
if self.train_cfg.rcnn.with_reg:
bbox_pred_stop_grad, bbox_gt, bbox_weights = bbox_pred_stop_grad[ind], bbox_gt[ind], bbox_weights[ind]
loss_bbox['loss_reg'] = loss_bbox['loss_reg'] + weighted_smoothl1(
bbox_pred_stop_grad,
bbox_gt,
bbox_weights,
avg_factor=num)
losses.update(loss_bbox)
# mask head forward and loss
if self.with_mask:
pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results])
mask_feats = self.mask_roi_extractor(
x[:self.mask_roi_extractor.num_inputs], pos_rois)
mask_pred = self.mask_head(mask_feats)
mask_targets = self.mask_head.get_target(
sampling_results, gt_masks, self.train_cfg.rcnn)
pos_labels = torch.cat(
[res.pos_gt_labels for res in sampling_results])
loss_mask = self.mask_head.loss(mask_pred, mask_targets,
pos_labels)
losses.update(loss_mask)
return losses
def loss(self,
match_score,
shift,
IoUs2,
distances_xy,
ids,
id_weights,
reduce=True):
losses = dict()
if self.dynamic:
n = len(match_score)
x_n = [s.size(0) for s in match_score]
ids = torch.split(ids, x_n, dim=0)
loss_match = 0.
match_acc = 0.
n_total = 0
batch_size = len(ids)
for score, cur_ids, cur_weights in zip(match_score, ids,
id_weights):
valid_idx = torch.nonzero(cur_weights).squeeze()
valid_cls = torch.index_select(cur_ids, 0, valid_idx)
nonzero_ids = valid_cls.clone()
nonzero_ids[nonzero_ids > 0] = 1
#loss_iou = torch.sum((1-IoUs[valid_idx , valid_cls])*nonzero_ids)
#loss_iou = loss_iou/(torch.sum(nonzero_ids)+1e-7)
#print(valid_idx)
valid_iou = IoUs2[valid_idx, valid_cls]
distance_mean = torch.zeros(2,
device=torch.cuda.current_device())
for match_cls in valid_cls.unique():
if match_cls == 0:
continue
valid_iou[(valid_cls == match_cls) & (
valid_iou != valid_iou[valid_cls == match_cls][0])] = 0
valid_iou[valid_iou < 0.7] = 0
valid_xy = distances_xy[:, valid_idx, valid_cls]
#dif_x = valid_xy[0]-shift[0][0]
#dif_y = valid_xy[1]-shift[0][1]
#distance_mean = torch.sum(torch.sqrt(dif_x*dif_x + dif_y*dif_y) * valid_iou / (torch.sum(valid_iou)+1e-7))
if len(valid_idx.size()) == 0: continue
n_valid = valid_idx.size(0)
n_total += n_valid
loss_match += weighted_cross_entropy(score,
cur_ids,
cur_weights,
reduce=reduce)
match_acc += accuracy(torch.index_select(score, 0, valid_idx),
valid_cls) * n_valid
losses['loss_match'] = loss_match / n
if n_total > 0:
losses['match_acc'] = match_acc / n_total
else:
print('not dynamic')
if match_score is not None:
valid_idx = torch.nonzero(cur_weights).squeeze()
losses['loss_match'] = weighted_cross_entropy(match_score,
ids,
id_weights,
reduce=reduce)
losses['match_acc'] = accuracy(
torch.index_select(match_score, 0, valid_idx),
torch.index_select(ids, 0, valid_idx))
return losses
