def add_gt_proposals(self, proposals, targets):
"""
Arguments:
proposals: BoxList
targets: BoxList
"""
# Get the device we're operating on
device = proposals.bbox3d.device
gt_boxes = [target.copy_with_fields([]) for target in targets]
# later cat of bbox requires all fields to be present for all bbox
# so we need to add a dummy for objectness that's missing
for gt_box in gt_boxes:
gt_box.add_field("objectness",
torch.ones(len(gt_box), device=device))
gt_box.add_field("is_gt", torch.ones(len(gt_box), device=device))
gt_box.constants = proposals.constants
proposals.add_field("is_gt", torch.zeros(len(proposals),
device=device))
batch_size = proposals.batch_size()
proposals = proposals.seperate_examples()
proposals = [
cat_boxlist_3d((proposal, gt_box), per_example=False)
for proposal, gt_box in zip(proposals, gt_boxes)
]
proposals = cat_boxlist_3d(proposals, per_example=True)
return proposals
def show_one_type(eval_type):
indices = metric_inds[eval_type]
if eval_type == 'TP' and self.add_gt_proposals:
indices = indices[0:-gt_num]
n0 = indices.shape[0]
pro_ = proposals[indices]
objectness_ = pro_.get_field('objectness')
logits_ = pred_logits[indices]
labels_ = labels[indices]
err_ = torch.abs(logits_ - labels_)
err_num = err_.sum()
print(f"\n * * * * * * * * \n{eval_type} :{n0} err num: {err_num}")
print(f"objectness_:{objectness_}\n")
if n0 > 0:
roi_class_pred_ = roi_class_pred[indices[:, None],
labels_[:, None]]
#if eval_type != 'TP':
print(f"roi_class_pred_:\n{roi_class_pred_}")
if eval_type == 'FP':
pro_.show__together(targets)
pass
if eval_type == 'FN' or eval_type == 'TP':
map_inds_ = 7 * labels_[:, None] + torch.tensor(
[0, 1, 2, 3, 4, 5, 6], device=device)
roi_box_regression_ = box_regression[indices[:, None],
map_inds_]
roi_box = self.box_coder.decode(roi_box_regression_,
pro_.bbox3d)
tar_reg = regression_targets[indices]
#roi_box = self.box_coder.decode(tar_reg, pro_.bbox3d)
print(f"target reg: \n{tar_reg[0:3]}")
print(f"roi_reg: \n{roi_box_regression_[0:3]}")
roi_box[:, 0] += 15
roi_boxlist_ = pro_.copy()
roi_boxlist_.bbox3d = roi_box
targets_ = targets.copy()
targets_.bbox3d[:, 0] += 15
bs_ = cat_boxlist_3d([pro_, roi_boxlist_],
per_example=False)
tg_ = cat_boxlist_3d([targets, targets_], False)
bs_.show__together(tg_, twolabels=True)
pass
pass
def __call__(self,
class_logits,
box_regression,
corners_semantic,
targets=None):
"""
Computes the loss for Faster R-CNN.
This requires that the subsample method has been called beforehand.
Arguments:
class_logits: [n,class_num]
box_regression: class agnostic [n,7]; class specific: [n,7*num_classes]
corners_semantic: [n,8*2]
targets for debuging only: [BoxList3D] len = batch_size
Returns:
classification_loss (Tensor)
box_loss (Tensor)
"""
#class_logits = cat(class_logits, dim=0)
n = class_logits.shape[0]
assert n == box_regression.shape[0]
assert corners_semantic is None or n == corners_semantic.shape[0]
class_num = class_logits.shape[1]
if not hasattr(self, "_proposals"):
raise RuntimeError("subsample needs to be called before")
proposals = self._proposals
proposals = cat_boxlist_3d(proposals, per_example=True)
labels = proposals.get_field("labels")
regression_targets = proposals.get_field("regression_targets")
pro_bbox3ds = proposals.bbox3d
if not self.need_seperate:
classification_loss = F.cross_entropy(class_logits, labels)
box_loss, corner_loss = self.box_loss(labels, box_regression,
regression_targets,
pro_bbox3ds,
corners_semantic)
else:
classification_loss = self.seperate_classifier.roi_cross_entropy_seperated(
class_logits, labels, proposals)
box_loss, corner_loss = self.seperate_classifier.roi_box_loss_seperated(
self.box_loss,
labels,
box_regression,
regression_targets,
pro_bbox3ds=pro_bbox3ds,
corners_semantic=corners_semantic)
pass
if SHOW_ROI_CLASSFICATION:
self.show_roi_cls_regs(proposals, classification_loss, box_loss,
class_logits, targets, box_regression,
regression_targets)
return classification_loss, box_loss, corner_loss
def post_processor(self, class_logits, box_regression, proposals,
post_processor_fn):
proposals_g, sep_ids_g = self.seperate_proposals(proposals)
#for gi in range(self.group_num):
class_logits_g = self.seperate_pred_logits(class_logits, sep_ids_g)
box_regression_g = self.seperate_pred_box(box_regression, sep_ids_g)
results_g = []
for gi in range(self.group_num):
result_gi = post_processor_fn(
(class_logits_g[gi], box_regression_g[gi]), proposals_g[gi])
results_g.append(result_gi)
batch_size = len(proposals)
result = []
for b in range(batch_size):
for gi in range(self.group_num):
sep_l = results_g[gi][b].extra_fields['labels']
results_g[gi][b].extra_fields[
'labels'] = self.sep_labels_to_org_labels[gi][sep_l]
result_b = [results_g[gi][b] for gi in range(self.group_num)]
result_b = cat_boxlist_3d(result_b, per_example=False)
result.append(result_b)
#print(result[0].fields())
return result
def seperate_subsample(self, proposals, targets, subsample_fn):
proposals_g, _ = self.seperate_proposals(proposals)
self.targets_g = self.seperate_targets_and_update_labels(targets)
for gi in range(self.group_num):
proposals_g[gi] = subsample_fn(proposals_g[gi], self.targets_g[gi])
assert self.targets_g[gi][0].get_field(
'labels').max() < self.class_nums[gi]
bs = len(proposals)
proposals_out = []
for i in range(bs):
psi = [proposals_g[j][i] for j in range(self.group_num)]
proposals_out.append(cat_boxlist_3d(psi, per_example=False))
#assert self.targets_1[0].get_field('labels').max() <= self.num_classes1 - 1
if DEBUG and False:
show_box_fields(proposals, 'In')
show_box_fields(proposals_0, 'Sep0')
show_box_fields(proposals_1, 'Sep1')
show_box_fields(proposals_0_, 'subs0')
show_box_fields(proposals_1_, 'subs1')
show_box_fields(proposals_out, 'Out')
show_box_fields(self.targets_0, 'T0')
show_box_fields(self.targets_1, 'T1')
return proposals_out
def cat_boxlist_3d_seperated(self, bboxes_ls):
batch_size = bboxes_ls[0].batch_size()
m = len(bboxes_ls)
assert m == self.group_num
bboxes_ = [None] * m
for gi in range(m):
bboxes_ls[gi].add_field(
'sep_id',
torch.ones([len(bboxes_ls[gi])], dtype=torch.int32) * gi)
bboxes_[gi] = bboxes_ls[gi].seperate_examples()
bboxes_ls_new = []
for j in range(batch_size):
bboxes_ls_new.append(
cat_boxlist_3d([bboxes_[i][j] for i in range(m)],
per_example=False))
bboxes_ls_new_all = cat_boxlist_3d(bboxes_ls_new, per_example=True)
return bboxes_ls_new_all
def __call__(self, class_logits, box_regression, targets=None):
"""
Computes the loss for Faster R-CNN.
This requires that the subsample method has been called beforehand.
Arguments:
class_logits (list[Tensor])
box_regression (list[Tensor])
targets for debuging only
Returns:
classification_loss (Tensor)
box_loss (Tensor)
"""
class_logits = cat(class_logits, dim=0)
box_regression = cat(box_regression, dim=0)
if not hasattr(self, "_proposals"):
raise RuntimeError("subsample needs to be called before")
proposals = self._proposals
proposals = cat_boxlist_3d(proposals, per_example=True)
labels = proposals.get_field("labels")
regression_targets = proposals.get_field("regression_targets")
pro_bbox3ds = proposals.bbox3d
if not self.need_seperate:
classification_loss = F.cross_entropy(class_logits, labels)
box_loss = self.box_loss(labels, box_regression,
regression_targets, pro_bbox3ds)
else:
classification_loss = self.seperate_classifier.roi_cross_entropy_seperated(
class_logits, labels, proposals)
box_loss = self.seperate_classifier.roi_box_loss_seperated(
self.box_loss, labels, box_regression, regression_targets,
pro_bbox3ds)
if SHOW_ROI_CLASSFICATION:
self.show_roi_cls_regs(proposals, classification_loss, box_loss,
class_logits, targets, box_regression,
regression_targets)
return classification_loss, box_loss
def forward_for_single_feature_map(self,
anchors,
objectness,
box_regression,
targets=None):
"""
Arguments:
anchors: BoxList -> all examples within same batch are concated together
objectness: tensor of size N
box_regression: tensor of size N, 7
"""
device = objectness.device
assert objectness.shape[0] == box_regression.shape[0] == len(anchors)
examples_idxscope = anchors.examples_idxscope
batch_size = anchors.batch_size()
result = []
for bi in range(batch_size):
# split examples in the batch
s, e = examples_idxscope[bi]
if SHOW_PRO_NUMS:
print(f'\n\nRPN input anchor num: {e-s}')
objectness_i0 = objectness[s:e]
box_regression_i = box_regression[s:e]
# put in the same format as anchors
objectness_i1 = objectness_i0.sigmoid()
# only choose top 2000 proposals for nms
num_anchors = e - s
fpn_pre_nms_top_n = min(self.fpn_pre_nms_top_n, num_anchors)
objectness_i, topk_idx = objectness_i1.topk(fpn_pre_nms_top_n,
dim=0,
sorted=True)
#batch_idx = torch.arange(N, device=device)[:, None]
box_regression_i = box_regression_i[topk_idx]
if anchors.size3d is None:
pcl_size3d = None
else:
pcl_size3d = anchors.size3d[bi:bi + 1]
concat_anchors_i = anchors.bbox3d[s:e, :]
concat_anchors_i = concat_anchors_i[topk_idx]
# decode box_regression to get proposals
proposals_i = self.box_coder.decode(box_regression_i,
concat_anchors_i)
#*********************************************************************
# apply nms
examples_idxscope_new = torch.tensor([[0, proposals_i.shape[0]]])
boxlist = BoxList3D(proposals_i,
pcl_size3d,
mode="yx_zb",
examples_idxscope=examples_idxscope_new,
constants={'prediction': True})
boxlist.add_field("objectness", objectness_i)
boxlist.set_as_prediction()
if SHOW_RPN_OUT_BEFORE_NMS:
print(
f'\n\n------------------------------------\n RPN out before NMS '
)
boxlist.show_together(targets[bi])
boxlist.show_by_objectness(0.8, targets[bi])
#boxlist = boxlist.clip_to_pcl(remove_empty=False)
#boxlist = remove_small_boxes3d(boxlist, self.min_size)
if SHOW_PRO_NUMS:
print(f'before nms box num: {len(boxlist)}')
boxlist_new = boxlist_nms_3d(
boxlist,
self.nms_thresh,
nms_aug_thickness=self.nms_aug_thickness,
max_proposals=self.fpn_post_nms_top_n,
score_field="objectness",
flag='rpn_post',
)
result.append(boxlist_new)
if SHOW_PRO_NUMS:
print(f'RPN out, after nms box num: {len(boxlist_new)}\n\n')
if SHOW_NMS_OUT:
print(
f'\n\n------------------------------------\n RPN out after NMS '
)
print('inference_3d.py SHOW_NMS_OUT')
objectness_i_new = boxlist_new.get_field('objectness')
print(f"objectness: {objectness_i_new[0:10]}")
boxlist_new.show_by_objectness(0.8, targets[bi])
import pdb
pdb.set_trace() # XXX BREAKPOINT
pass
result = cat_boxlist_3d(result, per_example=True)
return result
def show_roi_cls_regs(self, proposals, classification_loss, box_loss,
class_logits, targets, box_regression,
regression_targets):
'''
From rpn nms: FP, FN, TP
ROI: (1)remove all FP (2) add all FN, (3) keep all TP
'''
assert proposals.batch_size() == 1
targets = cat_boxlist_3d(targets, per_example=True)
roi_class_pred = F.softmax(class_logits)
pred_logits = torch.argmax(class_logits, 1)
labels = proposals.get_field("labels")
metric_inds, metric_evals = proposals.metric_4areas(
self.low_threshold, self.high_threshold)
gt_num = len(targets)
device = class_logits.device
num_classes = class_logits.shape[1]
class_err = (labels != pred_logits).sum()
print(
'\n-----------------------------------------\n roi classificatio\n'
)
print(f"RPN_NMS: {metric_evals}")
print(
f"classification_loss:{classification_loss}, box_loss: {box_loss}")
def show_one_type(eval_type):
indices = metric_inds[eval_type]
if eval_type == 'TP' and self.add_gt_proposals:
indices = indices[0:-gt_num]
n0 = indices.shape[0]
pro_ = proposals[indices]
objectness_ = pro_.get_field('objectness')
logits_ = pred_logits[indices]
labels_ = labels[indices]
err_ = torch.abs(logits_ - labels_)
err_num = err_.sum()
print(f"\n * * * * * * * * \n{eval_type} :{n0} err num: {err_num}")
print(f"objectness_:{objectness_}\n")
if n0 > 0:
roi_class_pred_ = roi_class_pred[indices[:, None],
labels_[:, None]]
#if eval_type != 'TP':
print(f"roi_class_pred_:\n{roi_class_pred_}")
if eval_type == 'FP':
pro_.show__together(targets)
pass
if eval_type == 'FN' or eval_type == 'TP':
map_inds_ = 7 * labels_[:, None] + torch.tensor(
[0, 1, 2, 3, 4, 5, 6], device=device)
roi_box_regression_ = box_regression[indices[:, None],
map_inds_]
roi_box = self.box_coder.decode(roi_box_regression_,
pro_.bbox3d)
tar_reg = regression_targets[indices]
#roi_box = self.box_coder.decode(tar_reg, pro_.bbox3d)
print(f"target reg: \n{tar_reg[0:3]}")
print(f"roi_reg: \n{roi_box_regression_[0:3]}")
roi_box[:, 0] += 15
roi_boxlist_ = pro_.copy()
roi_boxlist_.bbox3d = roi_box
targets_ = targets.copy()
targets_.bbox3d[:, 0] += 15
bs_ = cat_boxlist_3d([pro_, roi_boxlist_],
per_example=False)
tg_ = cat_boxlist_3d([targets, targets_], False)
bs_.show__together(tg_, twolabels=True)
pass
pass
show_one_type('FP')
show_one_type('FN')
show_one_type('TP')
return