def draw_cropped_frustum(self, bboxes, rect, Trv2c, P2):
# Y = C(R @ (rect @ Trv2c @ X) + T)
# uv = [Y0/Y2, Y1/Y2]
self.boxes3d('cropped_frustums',
prep.random_crop_frustum(bboxes, rect, Trv2c, P2),
colors=GLColor.Write,
alpha=0.5)
def sample_all(self,
root_path,
gt_boxes,
gt_names,
num_point_features,
random_crop=False,
gt_group_ids=None,
rect=None,
Trv2c=None,
P2=None):
sampled_num_dict = {}
sample_num_per_class = []
for class_name, max_sample_num in zip(
self._sample_classes, # 组内采样类别与添加的个数,Car,15
self._sample_max_nums):
sampled_num = int(
max_sample_num -
np.sum([n == class_name
for n in gt_names])) # 真值有n个训练对象，需要采样15-n个
sampled_num = np.round(self._rate * sampled_num).astype(np.int64)
sampled_num_dict[class_name] = sampled_num # {类别：需要采样的数量}
sample_num_per_class.append(sampled_num) # 每个类别需要采样的数量
sampled_groups = self._sample_classes # Car
if self._use_group_sampling: # False
assert gt_group_ids is not None
sampled_groups = []
sample_num_per_class = []
for group_name, class_names in self._group_name_to_names:
sampled_nums_group = [sampled_num_dict[n] for n in class_names]
sampled_num = np.max(sampled_nums_group)
sample_num_per_class.append(sampled_num)
sampled_groups.append(group_name)
total_group_ids = gt_group_ids
sampled = []
sampled_gt_boxes = []
avoid_coll_boxes = gt_boxes # 避免采样碰撞，这里是所有类别的真值
for class_name, sampled_num in zip(
sampled_groups, # 目标类别，采样数量
sample_num_per_class):
if sampled_num > 0:
if self._use_group_sampling: # False
sampled_cls = self.sample_group(class_name, sampled_num,
avoid_coll_boxes,
total_group_ids)
else:
sampled_cls = self.sample_class_v2(
class_name,
sampled_num, # 根据需求采样，返回不冲突的采样框的信息
avoid_coll_boxes)
sampled += sampled_cls
if len(sampled_cls) > 0: # 逐行存储采样对象的3d真值框信息
if len(sampled_cls) == 1:
sampled_gt_box = sampled_cls[0]["box3d_lidar"][
np.newaxis, ...]
else:
sampled_gt_box = np.stack(
[s["box3d_lidar"] for s in sampled_cls], axis=0)
sampled_gt_boxes += [sampled_gt_box] # 所有新增采样框
avoid_coll_boxes = np.concatenate(
[avoid_coll_boxes, sampled_gt_box],
axis=0) # 这是增加采样后所有的真值框
if self._use_group_sampling: # False
if len(sampled_cls) == 1:
sampled_group_ids = np.array(
sampled_cls[0]["group_id"])[np.newaxis, ...]
else:
sampled_group_ids = np.stack(
[s["group_id"] for s in sampled_cls], axis=0)
total_group_ids = np.concatenate(
[total_group_ids, sampled_group_ids], axis=0)
if len(sampled) > 0:
sampled_gt_boxes = np.concatenate(sampled_gt_boxes,
axis=0) # 所有采样框信息按行排列
num_sampled = len(sampled)
s_points_list = []
for info in sampled: # 根据采样对象读取对应点云信息，并加进当前帧数据中
s_points = np.fromfile(str(
pathlib.Path(root_path) / info["path"]),
dtype=np.float32)
s_points = s_points.reshape([-1, num_point_features])
# if not add_rgb_to_points:
# s_points = s_points[:, :4]
if "rot_transform" in info: # False
rot = info["rot_transform"]
s_points[:, :3] = box_np_ops.rotation_points_single_angle(
s_points[:, :3], rot, axis=2)
s_points[:, :3] += info["box3d_lidar"][:
3] # 将归一化的点云坐标恢复为原始点云坐标
s_points_list.append(s_points) # 采样的点云数据（numpy数组）组织成列表
# print(pathlib.Path(info["path"]).stem)
# gt_bboxes = np.stack([s["bbox"] for s in sampled], axis=0)
# if np.random.choice([False, True], replace=False, p=[0.3, 0.7]):
# do random crop.
if random_crop: # False
s_points_list_new = []
gt_bboxes = box_np_ops.box3d_to_bbox(sampled_gt_boxes, rect,
Trv2c, P2)
crop_frustums = prep.random_crop_frustum(
gt_bboxes, rect, Trv2c, P2)
for i in range(crop_frustums.shape[0]):
s_points = s_points_list[i]
mask = prep.mask_points_in_corners(
s_points, crop_frustums[i:i + 1]).reshape(-1)
num_remove = np.sum(mask)
if num_remove > 0 and (s_points.shape[0] -
num_remove) > 15:
s_points = s_points[np.logical_not(mask)]
s_points_list_new.append(s_points)
s_points_list = s_points_list_new
ret = {
"gt_names":
np.array([s["name"] for s in sampled]), # 采样对象类别，一维数组
"difficulty":
np.array([s["difficulty"] for s in sampled]), # 采样对象困难度，一维数组
"gt_boxes": sampled_gt_boxes, # 按行排列的采样真值框信息
"points": np.concatenate(s_points_list,
axis=0), # 点云是无序的，可以将不同对象的点云数据放在一起
"gt_masks": np.ones((num_sampled, ), dtype=np.bool_) # 采样框的标志
}
if self._use_group_sampling: # False
ret["group_ids"] = np.array([s["group_id"] for s in sampled])
else:
ret["group_ids"] = np.arange(gt_boxes.shape[0],
gt_boxes.shape[0] +
len(sampled)) # 采样对象序号
else:
ret = None
return ret
def sample_all(self,
root_path,
gt_boxes,
gt_names,
num_point_features,
random_crop=False,
gt_group_ids=None,
rect=None,
Trv2c=None,
P2=None):
sampled_num_dict = {}
sample_num_per_class = []
for class_name, max_sample_num in zip(self._sample_classes,
self._sample_max_nums):
sampled_num = int(max_sample_num -
np.sum([n == class_name for n in gt_names]))
sampled_num = np.round(self._rate * sampled_num).astype(np.int64)
sampled_num_dict[class_name] = sampled_num
sample_num_per_class.append(sampled_num)
sampled_groups = self._sample_classes
if self._use_group_sampling:
assert gt_group_ids is not None
sampled_groups = []
sample_num_per_class = []
for group_name, class_names in self._group_name_to_names:
sampled_nums_group = [sampled_num_dict[n] for n in class_names]
sampled_num = np.max(sampled_nums_group)
sample_num_per_class.append(sampled_num)
sampled_groups.append(group_name)
total_group_ids = gt_group_ids
sampled = []
sampled_gt_boxes = []
avoid_coll_boxes = gt_boxes
for class_name, sampled_num in zip(sampled_groups,
sample_num_per_class):
if sampled_num > 0:
if self._use_group_sampling:
sampled_cls = self.sample_group(class_name, sampled_num,
avoid_coll_boxes,
total_group_ids)
else:
sampled_cls = self.sample_class_v2(class_name, sampled_num,
avoid_coll_boxes)
sampled += sampled_cls
if len(sampled_cls) > 0:
if len(sampled_cls) == 1:
sampled_gt_box = sampled_cls[0]["box3d_lidar"][
np.newaxis, ...]
else:
sampled_gt_box = np.stack(
[s["box3d_lidar"] for s in sampled_cls], axis=0)
sampled_gt_boxes += [sampled_gt_box]
avoid_coll_boxes = np.concatenate(
[avoid_coll_boxes, sampled_gt_box], axis=0)
if self._use_group_sampling:
if len(sampled_cls) == 1:
sampled_group_ids = np.array(
sampled_cls[0]["group_id"])[np.newaxis, ...]
else:
sampled_group_ids = np.stack(
[s["group_id"] for s in sampled_cls], axis=0)
total_group_ids = np.concatenate(
[total_group_ids, sampled_group_ids], axis=0)
if len(sampled) > 0:
sampled_gt_boxes = np.concatenate(sampled_gt_boxes, axis=0)
num_sampled = len(sampled)
s_points_list = []
for info in sampled:
s_points = np.fromfile(str(
pathlib.Path(root_path) / info["path"]),
dtype=np.float32)
s_points = s_points.reshape([-1, num_point_features])
# if not add_rgb_to_points:
# s_points = s_points[:, :4]
if "rot_transform" in info:
rot = info["rot_transform"]
s_points[:, :3] = box_np_ops.rotation_points_single_angle(
s_points[:, :3], rot, axis=2)
s_points[:, :3] += info["box3d_lidar"][:3]
s_points_list.append(s_points)
# print(pathlib.Path(info["path"]).stem)
# gt_bboxes = np.stack([s["bbox"] for s in sampled], axis=0)
# if np.random.choice([False, True], replace=False, p=[0.3, 0.7]):
# do random crop.
if random_crop:
s_points_list_new = []
gt_bboxes = box_np_ops.box3d_to_bbox(sampled_gt_boxes, rect,
Trv2c, P2)
crop_frustums = prep.random_crop_frustum(
gt_bboxes, rect, Trv2c, P2)
for i in range(crop_frustums.shape[0]):
s_points = s_points_list[i]
mask = prep.mask_points_in_corners(
s_points, crop_frustums[i:i + 1]).reshape(-1)
num_remove = np.sum(mask)
if num_remove > 0 and (s_points.shape[0] -
num_remove) > 15:
s_points = s_points[np.logical_not(mask)]
s_points_list_new.append(s_points)
s_points_list = s_points_list_new
ret = {
"gt_names": np.array([s["name"] for s in sampled]),
"difficulty": np.array([s["difficulty"] for s in sampled]),
"gt_boxes": sampled_gt_boxes,
"points": np.concatenate(s_points_list, axis=0),
"gt_masks": np.ones((num_sampled, ), dtype=np.bool_)
}
if self._use_group_sampling:
ret["group_ids"] = np.array([s["group_id"] for s in sampled])
else:
ret["group_ids"] = np.arange(gt_boxes.shape[0],
gt_boxes.shape[0] + len(sampled))
else:
ret = None
return ret
def sample_all(self,
points,
voxel_size,
point_cloud_range,
voxel_grids,
root_path,
gt_boxes,
gt_names,
num_point_features,
random_crop=False,
gt_group_ids=None,
rect=None,
Trv2c=None,
P2=None):
sampled_num_dict = {}
sample_num_per_class = []
for class_name, max_sample_num in zip(self._sample_classes,
self._sample_max_nums):
sampled_num = int(max_sample_num -
np.sum([n == class_name for n in gt_names]))
sampled_num = np.round(self._rate * sampled_num).astype(np.int64)
sampled_num_dict[class_name] = sampled_num
sample_num_per_class.append(sampled_num)
sampled_groups = self._sample_classes
if self._use_group_sampling:
assert gt_group_ids is not None
sampled_groups = []
sample_num_per_class = []
for group_name, class_names in self._group_name_to_names:
sampled_nums_group = [sampled_num_dict[n] for n in class_names]
sampled_num = np.max(sampled_nums_group)
sample_num_per_class.append(sampled_num)
sampled_groups.append(group_name)
total_group_ids = gt_group_ids
sampled = []
sampled_gt_boxes = []
avoid_coll_boxes = gt_boxes
finetune_by_grd = False
voxel_scale = 3
voxel_size_scaled = voxel_size * voxel_scale
voxel_grids_scale = voxel_grids // voxel_scale
if finetune_by_grd:
grd_filter = ground_filter()
grd_filter.filter(points)
grd_gridmask = _points_to_gridmask_2d(grd_filter.ground_pc,
voxel_size_scaled,
point_cloud_range[:3],
voxel_grids_scale)
else:
grd_gridmask = None
for class_name, sampled_num in zip(sampled_groups,
sample_num_per_class):
if sampled_num > 0:
if finetune_by_grd:
assert self._use_group_sampling is not True
all_samples = self._sampler_dict[class_name].sample(
sampled_num * 15)
all_samples = copy.deepcopy(all_samples)
all_samples = _fine_sample_by_grd(all_samples,
grd_gridmask,
voxel_size_scaled,
point_cloud_range[:3],
voxel_grids_scale)
if len(all_samples) > sampled_num:
#print(len(all_samples), '>', sampled_num)
all_samples = all_samples[:sampled_num]
else:
all_samples = self._sampler_dict[class_name].sample(
sampled_num)
all_samples = copy.deepcopy(all_samples)
if self._use_group_sampling:
sampled_cls = self.sample_group(class_name, sampled_num,
avoid_coll_boxes,
total_group_ids)
else:
if len(all_samples) > 0:
sampled_cls = self.sample_class_v2(
all_samples, avoid_coll_boxes)
else:
sampled_cls = []
sampled += sampled_cls
if len(sampled_cls) > 0:
if len(sampled_cls) == 1:
sampled_gt_box = sampled_cls[0]["box3d_lidar"][
np.newaxis, ...]
else:
sampled_gt_box = np.stack(
[s["box3d_lidar"] for s in sampled_cls], axis=0)
sampled_gt_boxes += [sampled_gt_box]
avoid_coll_boxes = np.concatenate(
[avoid_coll_boxes, sampled_gt_box], axis=0)
if self._use_group_sampling:
if len(sampled_cls) == 1:
sampled_group_ids = np.array(
sampled_cls[0]["group_id"])[np.newaxis, ...]
else:
sampled_group_ids = np.stack(
[s["group_id"] for s in sampled_cls], axis=0)
total_group_ids = np.concatenate(
[total_group_ids, sampled_group_ids], axis=0)
if len(sampled) > 0:
sampled_gt_boxes = np.concatenate(sampled_gt_boxes, axis=0)
num_sampled = len(sampled)
s_points_list = []
for info in sampled:
s_points = np.fromfile(str(
pathlib.Path(root_path) / info["path"]),
dtype=np.float32)
s_points = s_points.reshape([-1, num_point_features])
# if not add_rgb_to_points:
# s_points = s_points[:, :4]
if "rot_transform" in info:
rot = info["rot_transform"]
s_points[:, :3] = box_np_ops.rotation_points_single_angle(
s_points[:, :3], rot, axis=2)
s_points[:, :3] += info["box3d_lidar"][:3]
s_points_list.append(s_points)
# print(pathlib.Path(info["path"]).stem)
# gt_bboxes = np.stack([s["bbox"] for s in sampled], axis=0)
# if np.random.choice([False, True], replace=False, p=[0.3, 0.7]):
# do random crop.
if random_crop:
s_points_list_new = []
gt_bboxes = box_np_ops.box3d_to_bbox(sampled_gt_boxes, rect,
Trv2c, P2)
crop_frustums = prep.random_crop_frustum(
gt_bboxes, rect, Trv2c, P2)
for i in range(crop_frustums.shape[0]):
s_points = s_points_list[i]
mask = prep.mask_points_in_corners(
s_points, crop_frustums[i:i + 1]).reshape(-1)
num_remove = np.sum(mask)
if num_remove > 0 and (s_points.shape[0] -
num_remove) > 15:
s_points = s_points[np.logical_not(mask)]
s_points_list_new.append(s_points)
s_points_list = s_points_list_new
finetune_samples_axis_z = True
if finetune_samples_axis_z:
gt_box_bottom_avg = np.mean(gt_boxes[:, 2])
# finetune boxes
sampled_gt_boxes_bottom = sampled_gt_boxes[:, 2]
delta = sampled_gt_boxes_bottom - gt_box_bottom_avg
sampled_gt_boxes[:, 2] -= delta
# finetune points
for i in range(len(s_points_list)):
s_points_list[i][:, 2] -= delta[i]
ret = {
"gt_names": np.array([s["name"] for s in sampled]),
"difficulty": np.array([s["difficulty"] for s in sampled]),
"gt_boxes": sampled_gt_boxes,
"points": np.concatenate(s_points_list, axis=0),
"gt_masks": np.ones((num_sampled, ), dtype=np.bool_)
}
if self._use_group_sampling:
ret["group_ids"] = np.array([s["group_id"] for s in sampled])
else:
ret["group_ids"] = np.arange(gt_boxes.shape[0],
gt_boxes.shape[0] + len(sampled))
else:
ret = None
return ret