def generate_gt_database(self):
gt_database = []
for idx, sample_id in enumerate(self.image_idx_list):
# sample_id = int(sample_id)
print('process gt sample (id=%s)' % sample_id)
pts_lidar = self.get_lidar(sample_id)
calib = self.get_calib(sample_id)
pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3])
pts_intensity = pts_lidar[:, 3]
obj_list = self.filtrate_objects(self.get_label(sample_id))
gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32)
for k, obj in enumerate(obj_list):
gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \
= obj.pos, obj.h, obj.w, obj.l, obj.ry
if gt_boxes3d.__len__() == 0:
print('No gt object')
if (self.split != "test"):
continue
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(
torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d))
for k in range(boxes_pts_mask_list.__len__()):
pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1)
cur_pts = pts_rect[pt_mask_flag].astype(np.float32)
cur_pts_intensity = pts_intensity[pt_mask_flag].astype(
np.float32)
if (self.split == "test"):
gt_box3d = None
else:
gt_box3d = gt_boxes3d[k]
sample_dict = {
'sample_id': sample_id,
'cls_type': obj_list[k].cls_type,
'gt_box3d': gt_boxes3d[k],
'points': cur_pts,
'intensity': cur_pts_intensity,
'obj': obj_list[k]
}
gt_database.append(sample_dict)
save_file_name = os.path.join(
args.save_dir,
'%s_gt_database_3level_%s.joblib' % (args.split, self.classes[-1]))
with open(save_file_name, 'wb') as f:
joblib.dump(gt_database, f)
# loaded_model = joblib.load(filename)
# pickle.dump(gt_database, f)
self.gt_database = gt_database
print('Save refine training sample info file to %s' % save_file_name)
def generate_gt_database(self):
gt_database = []
for idx, data in enumerate(self.__handle):
sample_id = int(data["sample_id"])
# print('process gt sample (id=%06d)' % sample_id)
# pts_lidar = self.get_lidar(sample_id)
# calib = self.get_calib(sample_id)
# pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3])
pts_rect = data['points']
pts_intensity = np.ones((len(pts_rect), 1))
# obj_list = self.filtrate_objects(self.get_label(sample_id))
# gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32)
# for k, obj in enumerate(data["boxes"]):
# gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \
# = obj[:3], obj[5], obj[4], obj[3], obj[6]
gt_boxes3d = data["boxes"][:, [0, 1, 2, 5, 4, 3, 6]]
if gt_boxes3d.__len__() == 0:
print('No gt object')
continue
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(
torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d))
for k in range(boxes_pts_mask_list.__len__()):
pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1)
cur_pts = pts_rect[pt_mask_flag].astype(np.float32)
cur_pts_intensity = pts_intensity[pt_mask_flag].astype(
np.float32)
sample_dict = {
'sample_id': sample_id,
'cls_type': "Pedestrian",
'gt_box3d': gt_boxes3d[k],
'points': cur_pts,
'intensity': cur_pts_intensity,
'obj': data["boxes"][k]
}
gt_database.append(sample_dict)
save_file_name = os.path.join(
args.save_dir,
'%s_gt_database_3level_%s.pkl' % (args.split, self.classes[-1]))
with open(save_file_name, 'wb') as f:
pickle.dump(gt_database, f)
self.gt_database = gt_database
print('Save refine training sample info file to %s' % save_file_name)
def generate_gt_database(self):
gt_database = []
#### print(self.image_idx_list) : data/KITTI/ImageSets train.txt
#### >> ['000000', '000003', '000007', '000009', '000010', '000011', '000012', '000013', '000014', '000016', ...]
for idx, sample_id in enumerate(self.image_idx_list):
sample_id = int(sample_id)
# print('process gt sample (id=%06d)' % sample_id)
pts_lidar = self.get_lidar(sample_id)
calib = self.get_calib(sample_id)
pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3])
pts_intensity = pts_lidar[:, 3]
obj_list = self.filtrate_objects(self.get_label(sample_id))
#### Try it!
gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32)
for k, obj in enumerate(obj_list):
gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \
= obj.pos, obj.h, obj.w, obj.l, obj.ry
# if gt_boxes3d.__len__() == 0:
# print('No gt object')
# continue
# else:
# print('gt object detected!')
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d))
for k in range(boxes_pts_mask_list.__len__()):
pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1)
cur_pts = pts_rect[pt_mask_flag].astype(np.float32)
cur_pts_intensity = pts_intensity[pt_mask_flag].astype(np.float32)
sample_dict = {'sample_id': sample_id,
'cls_type': obj_list[k].cls_type,
'gt_box3d': gt_boxes3d[k],
'points': cur_pts,
'intensity': cur_pts_intensity,
'obj': obj_list[k]}
gt_database.append(sample_dict)
#### print(gt_database)
save_file_name = os.path.join(args.save_dir, '%s_gt_database_3level_Multiclass.pkl' % (args.split)) #### Car -> Multiclass
with open(save_file_name, 'wb') as f:
pickle.dump(gt_database, f)
self.gt_database = gt_database
print('Save refine training sample info file to %s' % save_file_name)
def aug_one_scene(self, sample_id, pts_rect, pts_intensity, all_gt_boxes3d):
"""
:param pts_rect: (N, 3)
:param gt_boxes3d: (M1, 7)
:param all_gt_boxex3d: (M2, 7)
:return:
"""
assert self.gt_database is not None
extra_gt_num = np.random.randint(10, 15)
try_times = 50
cnt = 0
cur_gt_boxes3d = all_gt_boxes3d.copy()
cur_gt_boxes3d[:, 4] += 0.5
cur_gt_boxes3d[:, 5] += 0.5 # enlarge new added box to avoid too nearby boxes
extra_gt_obj_list = []
extra_gt_boxes3d_list = []
new_pts_list, new_pts_intensity_list = [], []
src_pts_flag = np.ones(pts_rect.shape[0], dtype=np.int32)
road_plane = self.get_road_plane(sample_id)
a, b, c, d = road_plane
while try_times > 0:
try_times -= 1
rand_idx = np.random.randint(0, self.gt_database.__len__() - 1)
new_gt_dict = self.gt_database[rand_idx]
new_gt_box3d = new_gt_dict['gt_box3d'].copy()
new_gt_points = new_gt_dict['points'].copy()
new_gt_intensity = new_gt_dict['intensity'].copy()
new_gt_obj = new_gt_dict['obj']
center = new_gt_box3d[0:3]
if PC_REDUCE_BY_RANGE and (self.check_pc_range(center) is False):
continue
if cnt > extra_gt_num:
break
if new_gt_points.__len__() < 5: # too few points
continue
# put it on the road plane
cur_height = (-d - a * center[0] - c * center[2]) / b
move_height = new_gt_box3d[1] - cur_height
new_gt_box3d[1] -= move_height
new_gt_points[:, 1] -= move_height
cnt += 1
iou3d = iou3d_utils.boxes_iou3d_gpu(torch.from_numpy(new_gt_box3d.reshape(1, 7)).cuda(),
torch.from_numpy(cur_gt_boxes3d).cuda()).cpu().numpy()
valid_flag = iou3d.max() < 1e-8
if not valid_flag:
continue
enlarged_box3d = new_gt_box3d.copy()
enlarged_box3d[3] += 2 # remove the points above and below the object
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(pts_rect),
torch.from_numpy(enlarged_box3d.reshape(1, 7)))
pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1)
src_pts_flag[pt_mask_flag] = 0 # remove the original points which are inside the new box
new_pts_list.append(new_gt_points)
new_pts_intensity_list.append(new_gt_intensity)
enlarged_box3d = new_gt_box3d.copy()
enlarged_box3d[4] += 0.5
enlarged_box3d[5] += 0.5 # enlarge new added box to avoid too nearby boxes
cur_gt_boxes3d = np.concatenate((cur_gt_boxes3d, enlarged_box3d.reshape(1, 7)), axis=0)
extra_gt_boxes3d_list.append(new_gt_box3d.reshape(1, 7))
extra_gt_obj_list.append(new_gt_obj)
if new_pts_list.__len__() == 0:
return False, pts_rect, pts_intensity, None, None
extra_gt_boxes3d = np.concatenate(extra_gt_boxes3d_list, axis=0)
# remove original points and add new points
pts_rect = pts_rect[src_pts_flag == 1]
pts_intensity = pts_intensity[src_pts_flag == 1]
new_pts_rect = np.concatenate(new_pts_list, axis=0)
new_pts_intensity = np.concatenate(new_pts_intensity_list, axis=0)
pts_rect = np.concatenate((pts_rect, new_pts_rect), axis=0)
pts_intensity = np.concatenate((pts_intensity, new_pts_intensity), axis=0)
return True, pts_rect, pts_intensity, extra_gt_boxes3d, extra_gt_obj_list
def get_rcnn_sample_info(self, roi_info):
sample_id, gt_box3d = roi_info['sample_id'], roi_info['gt_box3d']
rpn_xyz, rpn_features, rpn_intensity, seg_mask = self.rpn_feature_list[sample_id]
# augmentation original roi by adding noise
roi_box3d = self.aug_roi_by_noise(roi_info)
# point cloud pooling based on roi_box3d
pooled_boxes3d = kitti_utils.enlarge_box3d(roi_box3d.reshape(1, 7), cfg.RCNN.POOL_EXTRA_WIDTH)
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(rpn_xyz),
torch.from_numpy(pooled_boxes3d))
pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1)
cur_pts = rpn_xyz[pt_mask_flag].astype(np.float32)
# data augmentation
aug_pts = cur_pts.copy()
aug_gt_box3d = gt_box3d.copy().astype(np.float32)
aug_roi_box3d = roi_box3d.copy()
if cfg.AUG_DATA and self.mode == 'TRAIN':
# calculate alpha by ry
temp_boxes3d = np.concatenate([aug_roi_box3d.reshape(1, 7), aug_gt_box3d.reshape(1, 7)], axis=0)
temp_x, temp_z, temp_ry = temp_boxes3d[:, 0], temp_boxes3d[:, 2], temp_boxes3d[:, 6]
temp_beta = np.arctan2(temp_z, temp_x).astype(np.float64)
temp_alpha = -np.sign(temp_beta) * np.pi / 2 + temp_beta + temp_ry
# data augmentation
aug_pts, aug_boxes3d, aug_method = self.data_augmentation(aug_pts, temp_boxes3d, temp_alpha, mustaug=True, stage=2)
aug_roi_box3d, aug_gt_box3d = aug_boxes3d[0], aug_boxes3d[1]
aug_gt_box3d = aug_gt_box3d.astype(gt_box3d.dtype)
# Pool input points
valid_mask = 1 # whether the input is valid
if aug_pts.shape[0] == 0:
pts_features = np.zeros((1, 128), dtype=np.float32)
input_channel = 3 + int(cfg.RCNN.USE_INTENSITY) + int(cfg.RCNN.USE_MASK) + int(cfg.RCNN.USE_DEPTH)
pts_input = np.zeros((1, input_channel), dtype=np.float32)
valid_mask = 0
else:
pts_features = rpn_features[pt_mask_flag].astype(np.float32)
pts_intensity = rpn_intensity[pt_mask_flag].astype(np.float32)
pts_input_list = [aug_pts, pts_intensity.reshape(-1, 1)]
if cfg.RCNN.USE_INTENSITY:
pts_input_list = [aug_pts, pts_intensity.reshape(-1, 1)]
else:
pts_input_list = [aug_pts]
if cfg.RCNN.USE_MASK:
if cfg.RCNN.MASK_TYPE == 'seg':
pts_mask = seg_mask[pt_mask_flag].astype(np.float32)
elif cfg.RCNN.MASK_TYPE == 'roi':
pts_mask = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(aug_pts),
torch.from_numpy(aug_roi_box3d.reshape(1, 7)))
pts_mask = (pts_mask[0].numpy() == 1).astype(np.float32)
else:
raise NotImplementedError
pts_input_list.append(pts_mask.reshape(-1, 1))
if cfg.RCNN.USE_DEPTH:
pts_depth = np.linalg.norm(aug_pts, axis=1, ord=2)
pts_depth_norm = (pts_depth / 70.0) - 0.5
pts_input_list.append(pts_depth_norm.reshape(-1, 1))
pts_input = np.concatenate(pts_input_list, axis=1) # (N, C)
aug_gt_corners = kitti_utils.boxes3d_to_corners3d(aug_gt_box3d.reshape(-1, 7))
aug_roi_corners = kitti_utils.boxes3d_to_corners3d(aug_roi_box3d.reshape(-1, 7))
iou3d = kitti_utils.get_iou3d(aug_roi_corners, aug_gt_corners)
cur_iou = iou3d[0][0]
# regression valid mask
reg_valid_mask = 1 if cur_iou >= cfg.RCNN.REG_FG_THRESH and valid_mask == 1 else 0
# classification label
cls_label = 1 if cur_iou > cfg.RCNN.CLS_FG_THRESH else 0
if cfg.RCNN.CLS_BG_THRESH < cur_iou < cfg.RCNN.CLS_FG_THRESH or valid_mask == 0:
cls_label = -1
# canonical transform and sampling
pts_input_ct, gt_box3d_ct = self.canonical_transform(pts_input, aug_roi_box3d, aug_gt_box3d)
pts_input_ct, pts_features = self.rcnn_input_sample(pts_input_ct, pts_features)
sample_info = {'sample_id': sample_id,
'pts_input': pts_input_ct,
'pts_features': pts_features,
'cls_label': cls_label,
'reg_valid_mask': reg_valid_mask,
'gt_boxes3d_ct': gt_box3d_ct,
'roi_boxes3d': aug_roi_box3d,
'roi_size': aug_roi_box3d[3:6],
'gt_boxes3d': aug_gt_box3d}
return sample_info
def apply_gt_aug_to_one_scene(self, sample_id, pts_rect, pts_intensity, all_gt_boxes3d):
"""
:param pts_rect: (N, 3)
:param all_gt_boxex3d: (M2, 7)
:return:
"""
assert self.gt_database is not None
# extra_gt_num = np.random.randint(10, 15)
# try_times = 50
if cfg.GT_AUG_RAND_NUM:
extra_gt_num = np.random.randint(10, cfg.GT_EXTRA_NUM)
else:
extra_gt_num = cfg.GT_EXTRA_NUM
try_times = 100
cnt = 0
cur_gt_boxes3d = all_gt_boxes3d.copy()
cur_gt_boxes3d[:, 4] += 0.5 # TODO: consider different objects
cur_gt_boxes3d[:, 5] += 0.5 # enlarge new added box to avoid too nearby boxes
cur_gt_corners = kitti_utils.boxes3d_to_corners3d(cur_gt_boxes3d)
extra_gt_obj_list = []
extra_gt_boxes3d_list = []
new_pts_list, new_pts_intensity_list = [], []
src_pts_flag = np.ones(pts_rect.shape[0], dtype=np.int32)
road_plane = self.get_road_plane(sample_id)
a, b, c, d = road_plane
while try_times > 0:
if cnt > extra_gt_num:
break
try_times -= 1
if cfg.GT_AUG_HARD_RATIO > 0:
p = np.random.rand()
if p > cfg.GT_AUG_HARD_RATIO:
# use easy sample
rand_idx = np.random.randint(0, len(self.gt_database[0]))
new_gt_dict = self.gt_database[0][rand_idx]
else:
# use hard sample
rand_idx = np.random.randint(0, len(self.gt_database[1]))
new_gt_dict = self.gt_database[1][rand_idx]
else:
rand_idx = np.random.randint(0, self.gt_database.__len__())
new_gt_dict = self.gt_database[rand_idx]
new_gt_box3d = new_gt_dict['gt_box3d'].copy()
new_gt_points = new_gt_dict['points'].copy()
new_gt_intensity = new_gt_dict['intensity'].copy()
new_gt_obj = new_gt_dict['obj']
center = new_gt_box3d[0:3]
if cfg.PC_REDUCE_BY_RANGE and (self.check_pc_range(center) is False):
continue
if new_gt_points.__len__() < 5: # too few points
continue
# put it on the road plane
cur_height = (-d - a * center[0] - c * center[2]) / b
move_height = new_gt_box3d[1] - cur_height
new_gt_box3d[1] -= move_height
new_gt_points[:, 1] -= move_height
new_gt_obj.pos[1] -= move_height
new_enlarged_box3d = new_gt_box3d.copy()
new_enlarged_box3d[4] += 0.5
new_enlarged_box3d[5] += 0.5 # enlarge new added box to avoid too nearby boxes
cnt += 1
new_corners = kitti_utils.boxes3d_to_corners3d(new_enlarged_box3d.reshape(1, 7))
iou3d = kitti_utils.get_iou3d(new_corners, cur_gt_corners)
valid_flag = iou3d.max() < 1e-8
if not valid_flag:
continue
enlarged_box3d = new_gt_box3d.copy()
enlarged_box3d[3] += 2 # remove the points above and below the object
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(
torch.from_numpy(pts_rect), torch.from_numpy(enlarged_box3d.reshape(1, 7)))
pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1)
src_pts_flag[pt_mask_flag] = 0 # remove the original points which are inside the new box
new_pts_list.append(new_gt_points)
new_pts_intensity_list.append(new_gt_intensity)
cur_gt_boxes3d = np.concatenate((cur_gt_boxes3d, new_enlarged_box3d.reshape(1, 7)), axis=0)
cur_gt_corners = np.concatenate((cur_gt_corners, new_corners), axis=0)
extra_gt_boxes3d_list.append(new_gt_box3d.reshape(1, 7))
extra_gt_obj_list.append(new_gt_obj)
if new_pts_list.__len__() == 0:
return False, pts_rect, pts_intensity, None, None
extra_gt_boxes3d = np.concatenate(extra_gt_boxes3d_list, axis=0)
# remove original points and add new points
pts_rect = pts_rect[src_pts_flag == 1]
pts_intensity = pts_intensity[src_pts_flag == 1]
new_pts_rect = np.concatenate(new_pts_list, axis=0)
new_pts_intensity = np.concatenate(new_pts_intensity_list, axis=0)
pts_rect = np.concatenate((pts_rect, new_pts_rect), axis=0)
pts_intensity = np.concatenate((pts_intensity, new_pts_intensity), axis=0)
return True, pts_rect, pts_intensity, extra_gt_boxes3d, extra_gt_obj_list
def generate_gt_database(self):
gt_database = []
# read (have it as a reference) the KITTI dataset documentation.
# it is a text file in the library they have for handeling data.
# Download object development kit (1 MB) (including 3D object detection and bird's eye view evaluation code)
# http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d
# image_idx_list was defined in the parent
# it is a list of ids for data to choose for test/train
for idx, sample_id in enumerate(self.image_idx_list):
# get the id of the image/pointcloud (corresponding to 1 scene)
sample_id = int(sample_id)
print('process gt sample (id=%06d)' % sample_id)
# for later: There is the same number of images and point clouds so I think it is 1 image per point cloud
# read point cloud data
pts_lidar = self.get_lidar(sample_id)
calib = self.get_calib(sample_id)
# coordinate calibration on x,y,z (not intensity!) (rect stands for recalibrated ?!)
pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3])
# store intensity seperately
pts_intensity = pts_lidar[:, 3]
# first get a list of objects in the scene using the label data
# then only pick (filter) those that are in the self.classes
obj_list = self.filtrate_objects(self.get_label(sample_id))
# for each object (e.g. car) save the center (x,y,z) dimensions (w,h,l) and angle in the x-z plane (see paper figure3)
gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32)
for k, obj in enumerate(obj_list):
gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \
= obj.pos, obj.h, obj.w, obj.l, obj.ry
# if no objects of interest go to the next scene
if gt_boxes3d.__len__() == 0:
print('No gt object')
continue
# send the whole scene to 'pts_in_boxes3d_cpu' and the boxes in the scene
# determin which points are inside the box the returned value is a 2d array
# each row i is a mask of points that belong the box i and it is a pytorch tensor
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d))
# for each bounding box
# choose the points inside a bounding box and label them as a single instance
for k in range(boxes_pts_mask_list.__len__()):
# 'boxes_pts_mask_list[k]' is a pytorch tensor
pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1)
#choose points that are inside the current box ('pts_rect' is a numpy array)
cur_pts = pts_rect[pt_mask_flag].astype(np.float32)
cur_pts_intensity = pts_intensity[pt_mask_flag].astype(np.float32)
sample_dict = {'sample_id': sample_id, # scene id
'cls_type': obj_list[k].cls_type, # class (e.g. Car)
'gt_box3d': gt_boxes3d[k], # bounding box
'points': cur_pts, # point coords
'intensity': cur_pts_intensity,
'obj': obj_list[k]} # this includes all the other details we don't have above (as an object with attributes)
gt_database.append(sample_dict)
save_file_name = os.path.join(args.save_dir, '%s_gt_database_3level_%s.pkl' % (args.split, self.classes[-1]))
with open(save_file_name, 'wb') as f:
pickle.dump(gt_database, f)
self.gt_database = gt_database
print('Save refine training sample info file to %s' % save_file_name)
def roi_based_sample(pts_rect, pts_intensity, prev_pts_rect,
prev_pts_intensity, roi_scores, roi_boxes3d, ratio,
threshold):
"""
Samples points from pts_rect and pts_intensity based on the entropy gained
as shown in roi_scores and roi_boxes3d. Uses points found in last pass and adds to them
:param pts_rect: (N, 3)
:param pts_intensity: (N, 1)
:param prev_pts_rect: (N, 3)
:param prev_pts_intensity: (N, 1)
:param roi_scores: (M)
:param roi_boxes3d: (M, 7)
:param ratio: ratio of total pts_rect to return
:param threshold: rois below this threshold will not be observed
:return: pts_rect, pts_intensity
"""
# send everything to cpu
roi_scores = roi_scores.cpu()
roi_boxes3d = roi_boxes3d.cpu()
pts_and_intensity = np.append(pts_rect, pts_intensity, axis=1)
prev_pts_and_intensity = np.append(prev_pts_rect,
prev_pts_intensity,
axis=1)
# Sets of tuples of points to calculate diff
prev_pts_set = set(map(tuple, prev_pts_and_intensity))
pts_set = set(map(tuple, pts_and_intensity))
# Get pts not in previous pass
pts_diff_set = pts_set - prev_pts_set
pts_diff = np.array(list(pts_diff_set))
# pts and intensity diff
pts_rect_diff = pts_diff[:, :3]
# get entropy
confidence = (1 / (1 + np.exp(-roi_scores.numpy())))
entropy = das_utils.confidence_to_entropy(confidence)
# preen elements below threshold
threshold_mask = entropy > threshold
entropy = entropy[threshold_mask]
roi_scores = roi_scores[threshold_mask]
roi_boxes3d = roi_boxes3d[threshold_mask]
# print(len(entropy), len(roi_scores), len(roi_boxes3d))
normalized_entropy_distribution = (1 / sum(entropy)) * entropy
seen = set()
point_sets = list(range(len(roi_scores)))
# Changed to False reverse temporarily to see the difference
highest_to_lowest_entropy = sorted(range(len(entropy)),
key=entropy.__getitem__,
reverse=True)
# Points in boxes in order of decreasing entropy to prevent duplicates
for i in highest_to_lowest_entropy:
# Add new sampled points to set
pts_in_box_mask = pts_in_boxes3d_cpu(torch.Tensor(pts_rect_diff),
roi_boxes3d[i].unsqueeze(0))[0]
pts_in_box = pts_diff[pts_in_box_mask]
for pt in pts_in_box:
if tuple(pt) in seen:
np.delete(pts_in_box, np.argwhere(pts_in_box == pt))
else:
seen.add(tuple(pt))
point_sets[i] = pts_in_box
# Sampled points (add prev points)
sampled_points = prev_pts_and_intensity
# Fill with random points if less points than desired
if (len(pts_rect) * ratio > sum(
[len(point_sets[i]) for i in range(len(point_sets))])):
new_sampled_points = pts_and_intensity[pts_in_boxes3d_cpu(
torch.Tensor(pts_rect), roi_boxes3d)[0]]
shuffled_pts = np.random.permutation(pts_and_intensity)
i = 0
j = int(len(pts_rect) * ratio) - len(new_sampled_points)
while i < j:
if tuple(shuffled_pts[i]) in seen:
j += 1
else:
new_sampled_points = np.concatenate(
(new_sampled_points, shuffled_pts[i].reshape(1, 4)),
axis=0)
i += 1
sampled_points = np.concatenate((sampled_points, new_sampled_points),
axis=0)
return np.array(sampled_points)[:, :3], np.array(sampled_points)[:, 3]
# Sampling stage
# Flattened indices to sample from
indices = np.array([(i, j) for i in range(len(point_sets))
for j in range(len(point_sets[i]))])
# Probabilities for all of the indices
probs = np.array([
normalized_entropy_distribution[i] / len(point_sets[i])
for i in range(len(point_sets)) for j in range(len(point_sets[i]))
])
probs /= probs.sum()
sampled_indices_indices = np.random.choice(range(len(indices)),
size=int(ratio * len(pts_rect)),
p=probs,
replace=False)
sampled_indices = indices[sampled_indices_indices]
new_sampled_points = [
point_sets[idx[0]][idx[1]] for idx in sampled_indices
]
sampled_points = np.concatenate((sampled_points, new_sampled_points),
axis=0)
# print(sampled_points)
return np.array(sampled_points)[:, :3], np.array(sampled_points)[:, 3]
def generate_gt_database(self):
gt_database = []
for idx, sample_id in enumerate(self.image_idx_list):
sample_id = int(sample_id)
print('process gt sample (id=%06d)' % sample_id)
pts_lidar = self.get_lidar(sample_id)
calib = self.get_calib(sample_id)
pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3])
pts_intensity = pts_lidar[:, 3]
# (H,W,3)
img = self.get_image_rgb_with_normal(sample_id)
pts_img, pts_depth = calib.rect_to_img(pts_rect)
obj_list = self.filtrate_objects(self.get_label(sample_id))
gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32)
for k, obj in enumerate(obj_list):
gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \
= obj.pos, obj.h, obj.w, obj.l, obj.ry
if gt_boxes3d.__len__() == 0:
print('No gt object')
continue
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(
torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d))
shape = self.image_hw_with_padding_np.reshape((1, 1, 2))
for k in range(boxes_pts_mask_list.__len__()):
pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1)
cur_pts = pts_rect[pt_mask_flag].astype(np.float32)
cur_pts_intensity = pts_intensity[pt_mask_flag].astype(
np.float32)
# add img rbg into point cloud
# (N,2)
cur_pts_img_xy = pts_img[pt_mask_flag].astype(np.float32)
cur_pts_rgb = interpolate_img_by_xy(img, cur_pts_img_xy, shape)
sample_dict = {
'sample_id': sample_id,
'cls_type': obj_list[k].cls_type,
'gt_box3d': gt_boxes3d[k],
'points': cur_pts,
'rgb': cur_pts_rgb,
'intensity': cur_pts_intensity,
'obj': obj_list[k]
}
gt_database.append(sample_dict)
save_file_name = os.path.join(
args.save_dir,
'%s_gt_database_3level_%s.pkl' % (args.split, self.classes[-1]))
with open(save_file_name, 'wb') as f:
pickle.dump(gt_database, f)
self.gt_database = gt_database
print('Save refine training sample info file to %s' % save_file_name)
