def simple_test(self,
points,
img_metas,
imgs=None,
bev_seg_image=None,
rescale=False,
gt_bboxes_3d=None):
"""Test function without augmentaiton."""
# segmask_maps = self.generate_mask(points, vis_voxel_size=[0.16, 0.16, 4],
# vis_point_range=[0, -39.68, -3, 69.12, 39.68, 1],
# boxes=gt_bboxes_3d)
x, masks = self.extract_feat(points, img_metas)
outs = self.bbox_head(x)
# traced_script_module = torch.jit.trace(self.bbox_head, [x])
# traced_script_module.save("/home/zhangxiao/code/mmdetection3d/work_dirs/save_path/pts_bbox_head.zip")
bbox_list = self.bbox_head.get_bboxes(*outs,
img_metas,
rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
# bbox_list = self.bbox_head.get_bboxes(
# outs, img_metas, rescale=rescale)
# bbox_results = [
# bbox3d2result(bboxes, scores, labels)
# for bboxes, scores, labels in bbox_list
# ]
return bbox_results
def simple_test(self,
points,
img_metas,
imgs=None,
bev_seg_image=None,
rescale=False,
gt_bboxes_3d=None):
"""Test function without augmentaiton."""
# segmask_maps = self.generate_mask(points, vis_voxel_size=[0.16, 0.16, 4],
# vis_point_range=[0, -39.68, -3, 69.12, 39.68, 1],
# boxes=gt_bboxes_3d)
x = self.extract_feat(points, img_metas)
outs = self.bbox_head(x)
bbox_list = self.bbox_head.get_bboxes(*outs,
img_metas,
rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
# bbox_list = self.bbox_head.get_bboxes(
# outs, img_metas, rescale=rescale)
# bbox_results = [
# bbox3d2result(bboxes, scores, labels)
# for bboxes, scores, labels in bbox_list
# ]
return bbox_results
def simple_test_pts(self, x, img_metas, rescale=False):
"""Test function of point cloud branch."""
outs = self.pts_bbox_head(x)
bbox_list = self.pts_bbox_head.get_bboxes(
*outs, img_metas, rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
return bbox_results
def simple_test(self, points, img_metas, imgs=None, rescale=False):
"""Test function without augmentaiton."""
x = self.extract_feat(points, img_metas)
outs = self.bbox_head(x)
bbox_list = self.bbox_head.get_bboxes(
*outs, img_metas, rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
return bbox_results
def simple_test(self, points, img_metas, imgs=None, rescale=False):
"""Test function without augmentaiton."""
x = self.extract_feat(points, img_metas)
pred_dict = self.bbox_head(x)
bbox_list = self.bbox_head.get_bboxes(pred_dict, img_metas)
# print(len(bbox_list))
# print(bbox_list[0][3])
# print(type(bbox_list[0][0]))
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels, img_meta in bbox_list
]
return bbox_results[0] #list of dicts
def simple_test_pts(self, x, img_metas, rescale=False):
"""Test function of point cloud branch."""
outs = self.pts_bbox_head(x)
# traced_script_module = torch.jit.trace(self.pts_bbox_head, [x])
# traced_script_module.save("/home/zhangxiao/code/mmdetection3d/work_dirs/save_path/pts_bbox_head.zip")
bbox_list = self.pts_bbox_head.get_bboxes(*outs,
img_metas,
rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
return bbox_results
def simple_test_pts(self, x, img_metas, rescale=False):
"""Test function of point cloud branch."""
outs = self.pts_bbox_head(x)
bbox_list = self.pts_bbox_head.get_bboxes(*outs,
img_metas,
rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
# bbox_results[0].keys=(boxes_3d, scores_3d, labels_3d)
# values on cpu
# print(len(bbox_results)) # batch_size
return bbox_results
def simple_test_pts(self, pts_feats, img_metas):
"""Test function of point cloud branch."""
outs = self.pts_bbox_head(pts_feats)
bbox_list = self.pts_bbox_head.get_bboxes(*outs, img_metas)
# bbox_list: list of tuple; len(bbox_list) = test_batch_size
# bbox_list[0]: (bboxes, scores, labels)
# bboxes: LiDARInstance3DBoxes of tensor (N, 9); scores: tensor (N, ); labels: tensor (N, )
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
# bbox_results: [dict(boxes_3d=bboxes, scores_3d=scores, labels_3d=labels)] on cpu
# bboxes: LiDARInstance3DBoxes on cpu
# scores: tensor (N, ); labels: tensor(N, )
return bbox_results
def simple_test(self, img, img_metas):
"""Test without augmentations.
Args:
img (torch.Tensor): Input images of shape (N, C_in, H, W).
img_metas (list): Image metas.
Returns:
list[dict]: Predicted 3d boxes.
"""
x = self.extract_feat(img, img_metas)
x = self.bbox_head(x)
bbox_list = self.bbox_head.get_bboxes(*x, img_metas)
bbox_results = [
bbox3d2result(det_bboxes, det_scores, det_labels)
for det_bboxes, det_scores, det_labels in bbox_list
]
return bbox_results
def simple_test(self, img, img_metas, rescale=False):
"""Test function without test time augmentation.
Args:
imgs (list[torch.Tensor]): List of multiple images
img_metas (list[dict]): List of image information.
rescale (bool, optional): Whether to rescale the results.
Defaults to False.
Returns:
list[list[np.ndarray]]: BBox results of each image and classes.
The outer list corresponds to each image. The inner list
corresponds to each class.
"""
x = self.extract_feat(img)
outs = self.bbox_head(x)
bbox_outputs = self.bbox_head.get_bboxes(*outs,
img_metas,
rescale=rescale)
if self.bbox_head.pred_bbox2d:
from mmdet.core import bbox2result
bbox2d_img = [
bbox2result(bboxes2d, labels, self.bbox_head.num_classes)
for bboxes, scores, labels, attrs, bboxes2d in bbox_outputs
]
bbox_outputs = [bbox_outputs[0][:-1]]
bbox_img = [
bbox3d2result(bboxes, scores, labels, attrs)
for bboxes, scores, labels, attrs in bbox_outputs
]
bbox_list = [dict() for i in range(len(img_metas))]
for result_dict, img_bbox in zip(bbox_list, bbox_img):
result_dict['img_bbox'] = img_bbox
if self.bbox_head.pred_bbox2d:
for result_dict, img_bbox2d in zip(bbox_list, bbox2d_img):
result_dict['img_bbox2d'] = img_bbox2d
return bbox_list
def simple_test(self, points, img_metas, imgs=None, rescale=False):
"""Forward of testing.
Args:
points (list[torch.Tensor]): Points of each sample.
img_metas (list): Image metas.
rescale (bool): Whether to rescale results.
Returns:
list: Predicted 3d boxes.
"""
points_cat = torch.stack(points)
x = self.extract_feat(points_cat)
bbox_preds = self.bbox_head(x, self.test_cfg.sample_mod)
bbox_list = self.bbox_head.get_bboxes(
points_cat, bbox_preds, img_metas, rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
return bbox_results
def simple_test(self, points, img_metas, imgs=None, rescale=False):
"""Test function without augmentaiton."""
x, ref_map = self.extract_feat(points, img_metas)
img_feat = self.img_encoder(imgs)
img_feat = self.bbox_head([img_feat])
outs = self.bbox_head(x)
#从反射率地图中求cls score
ref_cls_score = self.ref_to_cls(ref_map)
for i in range(len(outs)):
outs[i][0] += img_feat[i][0]
#用反射率加权
outs[0][0] *= (ref_cls_score * 2 + 1)
bbox_list = self.bbox_head.get_bboxes(*outs,
img_metas,
rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
return bbox_results
def _format_bbox(self, results, jsonfile_prefix=None):
"""Convert the results to the standard format.
Args:
results (list[dict]): Testing results of the dataset.
jsonfile_prefix (str): The prefix of the output jsonfile.
You can specify the output directory/filename by
modifying the jsonfile_prefix. Default: None.
Returns:
str: Path of the output json file.
"""
nusc_annos = {}
mapped_class_names = self.CLASSES
print('Start to convert detection format...')
CAM_NUM = 6
for sample_id, det in enumerate(mmcv.track_iter_progress(results)):
if sample_id % CAM_NUM == 0:
boxes_per_frame = []
attrs_per_frame = []
# need to merge results from images of the same sample
annos = []
boxes, attrs = output_to_nusc_box(det)
sample_token = self.data_infos[sample_id]['token']
boxes, attrs = cam_nusc_box_to_global(self.data_infos[sample_id],
boxes, attrs,
mapped_class_names,
self.eval_detection_configs,
self.eval_version)
boxes_per_frame.extend(boxes)
attrs_per_frame.extend(attrs)
# Remove redundant predictions caused by overlap of images
if (sample_id + 1) % CAM_NUM != 0:
continue
boxes = global_nusc_box_to_cam(
self.data_infos[sample_id + 1 - CAM_NUM], boxes_per_frame,
mapped_class_names, self.eval_detection_configs,
self.eval_version)
cam_boxes3d, scores, labels = nusc_box_to_cam_box3d(boxes)
# box nms 3d over 6 images in a frame
# TODO: move this global setting into config
nms_cfg = dict(use_rotate_nms=True,
nms_across_levels=False,
nms_pre=4096,
nms_thr=0.05,
score_thr=0.01,
min_bbox_size=0,
max_per_frame=500)
from mmcv import Config
nms_cfg = Config(nms_cfg)
cam_boxes3d_for_nms = xywhr2xyxyr(cam_boxes3d.bev)
boxes3d = cam_boxes3d.tensor
# generate attr scores from attr labels
attrs = labels.new_tensor([attr for attr in attrs_per_frame])
boxes3d, scores, labels, attrs = box3d_multiclass_nms(
boxes3d,
cam_boxes3d_for_nms,
scores,
nms_cfg.score_thr,
nms_cfg.max_per_frame,
nms_cfg,
mlvl_attr_scores=attrs)
cam_boxes3d = CameraInstance3DBoxes(boxes3d, box_dim=9)
det = bbox3d2result(cam_boxes3d, scores, labels, attrs)
boxes, attrs = output_to_nusc_box(det)
boxes, attrs = cam_nusc_box_to_global(
self.data_infos[sample_id + 1 - CAM_NUM], boxes, attrs,
mapped_class_names, self.eval_detection_configs,
self.eval_version)
for i, box in enumerate(boxes):
name = mapped_class_names[box.label]
attr = self.get_attr_name(attrs[i], name)
nusc_anno = dict(sample_token=sample_token,
translation=box.center.tolist(),
size=box.wlh.tolist(),
rotation=box.orientation.elements.tolist(),
velocity=box.velocity[:2].tolist(),
detection_name=name,
detection_score=box.score,
attribute_name=attr)
annos.append(nusc_anno)
# other views results of the same frame should be concatenated
if sample_token in nusc_annos:
nusc_annos[sample_token].extend(annos)
else:
nusc_annos[sample_token] = annos
nusc_submissions = {
'meta': self.modality,
'results': nusc_annos,
}
mmcv.mkdir_or_exist(jsonfile_prefix)
res_path = osp.join(jsonfile_prefix, 'results_nusc.json')
print('Results writes to', res_path)
mmcv.dump(nusc_submissions, res_path)
return res_path
def simple_test(self,
points=None,
img_metas=None,
img=None,
bboxes_2d=None,
rescale=False,
**kwargs):
"""Test without augmentation, stage 2.
Args:
points (list[torch.Tensor], optional): Elements in the list
should have a shape NxC, the list indicates all point-clouds
in the batch. Defaults to None.
img_metas (list[dict], optional): List indicates
images in a batch. Defaults to None.
img (torch.Tensor, optional): Should have a shape NxCxHxW,
which contains all images in the batch. Defaults to None.
bboxes_2d (list[torch.Tensor], optional):
Provided 2d bboxes, not supported yet. Defaults to None.
rescale (bool, optional): Whether or not rescale bboxes.
Defaults to False.
Returns:
list[dict]: Predicted 3d boxes.
"""
bboxes_2d = self.extract_bboxes_2d(
img, img_metas, train=False, bboxes_2d=bboxes_2d, **kwargs)
points = torch.stack(points)
seeds_3d, seed_3d_features, seed_indices = \
self.extract_pts_feat(points)
img_features, masks = self.fusion_layer(img, bboxes_2d, seeds_3d,
img_metas)
inds = sample_valid_seeds(masks, self.num_sampled_seed)
batch_size, img_feat_size = img_features.shape[:2]
pts_feat_size = seed_3d_features.shape[1]
inds_img = inds.view(batch_size, 1, -1).expand(-1, img_feat_size, -1)
img_features = img_features.gather(-1, inds_img)
inds = inds % inds.shape[1]
inds_seed_xyz = inds.view(batch_size, -1, 1).expand(-1, -1, 3)
seeds_3d = seeds_3d.gather(1, inds_seed_xyz)
inds_seed_feats = inds.view(batch_size, 1,
-1).expand(-1, pts_feat_size, -1)
seed_3d_features = seed_3d_features.gather(-1, inds_seed_feats)
seed_indices = seed_indices.gather(1, inds)
img_features = self.img_mlp(img_features)
fused_features = torch.cat([seed_3d_features, img_features], dim=1)
feat_dict = dict(
seed_points=seeds_3d,
seed_features=fused_features,
seed_indices=seed_indices)
bbox_preds = self.pts_bbox_head_joint(feat_dict,
self.test_cfg.pts.sample_mod)
bbox_list = self.pts_bbox_head_joint.get_bboxes(
points, bbox_preds, img_metas, rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
return bbox_results
def aug_test(self, imgs, img_metas, rescale=False):
"""Test function with test time augmentation."""
feats = self.extract_feats(imgs)
# only support aug_test for one sample
outs_list = [self.bbox_head(x) for x in feats]
for i, img_meta in enumerate(img_metas):
if img_meta[0]['pcd_horizontal_flip']:
for j in range(len(outs_list[i])): # for each prediction
if outs_list[i][j][0] is None:
continue
for k in range(len(outs_list[i][j])):
# every stride of featmap
outs_list[i][j][k] = torch.flip(outs_list[i][j][k],
dims=[3])
reg = outs_list[i][1]
for reg_feat in reg:
# offset_x
reg_feat[:, 0, :, :] = 1 - reg_feat[:, 0, :, :]
# velo_x
if self.bbox_head.pred_velo:
reg_feat[:, 7, :, :] = -reg_feat[:, 7, :, :]
# rotation
reg_feat[:, 6, :, :] = -reg_feat[:, 6, :, :] + np.pi
merged_outs = []
for i in range(len(outs_list[0])): # for each prediction
merged_feats = []
for j in range(len(outs_list[0][i])):
if outs_list[0][i][0] is None:
merged_feats.append(None)
continue
# for each stride of featmap
avg_feats = torch.mean(torch.cat([x[i][j] for x in outs_list]),
dim=0,
keepdim=True)
if i == 1: # regression predictions
# rot/velo/2d det keeps the original
avg_feats[:, 6:, :, :] = \
outs_list[0][i][j][:, 6:, :, :]
if i == 2:
# dir_cls keeps the original
avg_feats = outs_list[0][i][j]
merged_feats.append(avg_feats)
merged_outs.append(merged_feats)
merged_outs = tuple(merged_outs)
bbox_outputs = self.bbox_head.get_bboxes(*merged_outs,
img_metas[0],
rescale=rescale)
if self.bbox_head.pred_bbox2d:
from mmdet.core import bbox2result
bbox2d_img = [
bbox2result(bboxes2d, labels, self.bbox_head.num_classes)
for bboxes, scores, labels, attrs, bboxes2d in bbox_outputs
]
bbox_outputs = [bbox_outputs[0][:-1]]
bbox_img = [
bbox3d2result(bboxes, scores, labels, attrs)
for bboxes, scores, labels, attrs in bbox_outputs
]
bbox_list = dict()
bbox_list.update(img_bbox=bbox_img[0])
if self.bbox_head.pred_bbox2d:
bbox_list.update(img_bbox2d=bbox2d_img[0])
return [bbox_list]
def simple_test_pts(self,
pts_feats,
img_metas,
rescale=False,
voxel_feats=None,
voxel_coors=None,
point_feats=None,
point_coors=None,
pts_res_dict=None,
ori_points=None):
"""Test function of point cloud branch."""
pred_v_semantic_cls = self.pts_bbox_head(pts_feats,
mode='semantic',
voxel_feat=voxel_feats,
voxel_coors=voxel_coors)
pred_v_sem_labels = []
for temp in pred_v_semantic_cls:
pred_v_sem_labels.append(torch.argmax(temp, dim=1))
point_sem_labels = self.pts_bbox_head.get_point_label_from_voxel(
pred_v_sem_labels, voxel_coors, point_coors)
point_ins_labels, bbox_results = [None for _ in point_sem_labels
], [None for _ in point_sem_labels]
if self.need_instance:
instance_fm = pts_res_dict["low_reso_fm"]
voxel_sem_feats = None
if self.instance_head.use_sem_feats:
pred_v_semantic_cls = torch.cat(pred_v_semantic_cls, dim=0)
voxel_sem_feats = pred_v_semantic_cls.detach()[:, 1:]
voxel_sem_feats = F.softmax(voxel_sem_feats, dim=1)
# filter: reserve thing class
temp_v_sem_labels = torch.cat(pred_v_sem_labels, dim=0)
thing2point_labels = torch.tensor(
self.instance_head.thing2point_labels).type_as(
temp_v_sem_labels)
pred_v_thing_mask = (
temp_v_sem_labels[:,
None] == thing2point_labels[None, :]).any(1)
pts_res_dict['ins_voxel_mean'] = pts_res_dict['ins_voxel_mean'][
pred_v_thing_mask].view(-1,
pts_res_dict['ins_voxel_mean'].size(1))
pts_res_dict['ins_voxel_coors'] = pts_res_dict['ins_voxel_coors'][
pred_v_thing_mask].view(
-1, pts_res_dict['ins_voxel_coors'].size(1))
v_offset_pred_list, p_offset_pred_list, hm_preds, bbox_preds = \
self.instance_head(instance_fm,
voxel_feat=voxel_feats[pred_v_thing_mask].view(-1, voxel_feats.size(1)),
voxel_coors=voxel_coors[pred_v_thing_mask].view(-1, voxel_coors.size(1)),
voxel_sem_feats=voxel_sem_feats[pred_v_thing_mask].view(-1, voxel_sem_feats.size(1)))
pred_bboxes_list = [None for _ in point_sem_labels]
if self.instance_head.need_bbox:
bbox_preds_dict = self.instance_head.format_bbox_preds(
hm_preds, bbox_preds)
pred_bboxes_list = self.instance_head.get_bboxes(
bbox_preds_dict, img_metas)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in pred_bboxes_list
]
else:
bbox_results = [None for _ in point_sem_labels]
pred_v_ins_labels, pred_v_sem_labels, debug_v_shift_xyz_list, debug_valid_v_mask_list = \
self.instance_head.get_ins_label(
v_offset_pred_list[-1],
pts_res_dict['ins_voxel_mean'],
voxel_coors,
pred_bboxes_list,
pred_v_sem_labels,
pred_v_thing_mask)
point_ins_labels = self.pts_bbox_head.get_point_label_from_voxel(
pred_v_ins_labels, voxel_coors, point_coors)
if self.instance_head.test_cfg['refine_sem_labels']:
point_sem_labels = self.pts_bbox_head.get_point_label_from_voxel(
pred_v_sem_labels, voxel_coors, point_coors)
# debug
need_debug = False
if need_debug:
debug_shift_point_xyz = self.pts_bbox_head.get_point_label_from_voxel(
debug_v_shift_xyz_list, voxel_coors, point_coors)
debug_shift_point_mask = self.pts_bbox_head.get_point_label_from_voxel(
debug_valid_v_mask_list, voxel_coors, point_coors)
debug_shift_point_xyz = debug_shift_point_xyz[0][
debug_shift_point_mask[0]]
debug_shift_point_sem_label = point_sem_labels[0][
debug_shift_point_mask[0]]
debug_shift_point_ins_label = point_ins_labels[0][
debug_shift_point_mask[0]]
debug_pts = torch.cat(
[debug_shift_point_xyz, ori_points[0][:, :3]], dim=0)
debug_pts = torch.cat([debug_pts, debug_pts[:, :2]], dim=-1)
debug_sem_label = torch.cat([
debug_shift_point_sem_label,
debug_shift_point_sem_label.new_zeros(
ori_points[0].size(0))
])
debug_ins_label = torch.cat([
debug_shift_point_ins_label,
debug_shift_point_ins_label.new_zeros(
ori_points[0].size(0))
])
debug_label = torch.cat(
[debug_sem_label.view(-1, 1),
debug_ins_label.view(-1, 1)],
dim=-1)
debug_pts = debug_pts.cpu().numpy()
file_name = str(self.debug_global_id).zfill(6)
bin_file_path = '/home/radmin/jk/code/seg/SelectiveSeg/work_dirs/debug/lidar'
debug_pts.tofile(bin_file_path + '/' + file_name + '.bin')
debug_label = debug_label.cpu().numpy()
debug_label = np.array(debug_label, dtype=np.uint8)
label_file_path = '/home/radmin/jk/code/seg/SelectiveSeg/work_dirs/debug/label'
debug_label.tofile(label_file_path + '/' + file_name + '.bin')
self.debug_global_id = self.debug_global_id + 1
self.instance_head.debug_bbox_file_id = self.debug_global_id
'''
outs = self.pts_bbox_head(x)
bbox_list = self.pts_bbox_head.get_bboxes(
outs, img_metas, rescale=rescale)
bbox_results = [
bbox3d2result(bboxes, scores, labels)
for bboxes, scores, labels in bbox_list
]
'''
return point_sem_labels, point_ins_labels, bbox_results
