def draw_cropped_frustum(self, bboxes, rect, Trv2c, P2): # Y = C(R @ (rect @ Trv2c @ X) + T) # uv = [Y0/Y2, Y1/Y2] self.boxes3d('cropped_frustums', prep.random_crop_frustum(bboxes, rect, Trv2c, P2), colors=GLColor.Write, alpha=0.5)
def sample_all(self, root_path, gt_boxes, gt_names, num_point_features, random_crop=False, gt_group_ids=None, rect=None, Trv2c=None, P2=None): sampled_num_dict = {} sample_num_per_class = [] for class_name, max_sample_num in zip( self._sample_classes, # 组内采样类别与添加的个数,Car,15 self._sample_max_nums): sampled_num = int( max_sample_num - np.sum([n == class_name for n in gt_names])) # 真值有n个训练对象,需要采样15-n个 sampled_num = np.round(self._rate * sampled_num).astype(np.int64) sampled_num_dict[class_name] = sampled_num # {类别:需要采样的数量} sample_num_per_class.append(sampled_num) # 每个类别需要采样的数量 sampled_groups = self._sample_classes # Car if self._use_group_sampling: # False assert gt_group_ids is not None sampled_groups = [] sample_num_per_class = [] for group_name, class_names in self._group_name_to_names: sampled_nums_group = [sampled_num_dict[n] for n in class_names] sampled_num = np.max(sampled_nums_group) sample_num_per_class.append(sampled_num) sampled_groups.append(group_name) total_group_ids = gt_group_ids sampled = [] sampled_gt_boxes = [] avoid_coll_boxes = gt_boxes # 避免采样碰撞,这里是所有类别的真值 for class_name, sampled_num in zip( sampled_groups, # 目标类别,采样数量 sample_num_per_class): if sampled_num > 0: if self._use_group_sampling: # False sampled_cls = self.sample_group(class_name, sampled_num, avoid_coll_boxes, total_group_ids) else: sampled_cls = self.sample_class_v2( class_name, sampled_num, # 根据需求采样,返回不冲突的采样框的信息 avoid_coll_boxes) sampled += sampled_cls if len(sampled_cls) > 0: # 逐行存储采样对象的3d真值框信息 if len(sampled_cls) == 1: sampled_gt_box = sampled_cls[0]["box3d_lidar"][ np.newaxis, ...] else: sampled_gt_box = np.stack( [s["box3d_lidar"] for s in sampled_cls], axis=0) sampled_gt_boxes += [sampled_gt_box] # 所有新增采样框 avoid_coll_boxes = np.concatenate( [avoid_coll_boxes, sampled_gt_box], axis=0) # 这是增加采样后所有的真值框 if self._use_group_sampling: # False if len(sampled_cls) == 1: sampled_group_ids = np.array( sampled_cls[0]["group_id"])[np.newaxis, ...] else: sampled_group_ids = np.stack( [s["group_id"] for s in sampled_cls], axis=0) total_group_ids = np.concatenate( [total_group_ids, sampled_group_ids], axis=0) if len(sampled) > 0: sampled_gt_boxes = np.concatenate(sampled_gt_boxes, axis=0) # 所有采样框信息按行排列 num_sampled = len(sampled) s_points_list = [] for info in sampled: # 根据采样对象读取对应点云信息,并加进当前帧数据中 s_points = np.fromfile(str( pathlib.Path(root_path) / info["path"]), dtype=np.float32) s_points = s_points.reshape([-1, num_point_features]) # if not add_rgb_to_points: # s_points = s_points[:, :4] if "rot_transform" in info: # False rot = info["rot_transform"] s_points[:, :3] = box_np_ops.rotation_points_single_angle( s_points[:, :3], rot, axis=2) s_points[:, :3] += info["box3d_lidar"][: 3] # 将归一化的点云坐标恢复为原始点云坐标 s_points_list.append(s_points) # 采样的点云数据(numpy数组)组织成列表 # print(pathlib.Path(info["path"]).stem) # gt_bboxes = np.stack([s["bbox"] for s in sampled], axis=0) # if np.random.choice([False, True], replace=False, p=[0.3, 0.7]): # do random crop. if random_crop: # False s_points_list_new = [] gt_bboxes = box_np_ops.box3d_to_bbox(sampled_gt_boxes, rect, Trv2c, P2) crop_frustums = prep.random_crop_frustum( gt_bboxes, rect, Trv2c, P2) for i in range(crop_frustums.shape[0]): s_points = s_points_list[i] mask = prep.mask_points_in_corners( s_points, crop_frustums[i:i + 1]).reshape(-1) num_remove = np.sum(mask) if num_remove > 0 and (s_points.shape[0] - num_remove) > 15: s_points = s_points[np.logical_not(mask)] s_points_list_new.append(s_points) s_points_list = s_points_list_new ret = { "gt_names": np.array([s["name"] for s in sampled]), # 采样对象类别,一维数组 "difficulty": np.array([s["difficulty"] for s in sampled]), # 采样对象困难度,一维数组 "gt_boxes": sampled_gt_boxes, # 按行排列的采样真值框信息 "points": np.concatenate(s_points_list, axis=0), # 点云是无序的,可以将不同对象的点云数据放在一起 "gt_masks": np.ones((num_sampled, ), dtype=np.bool_) # 采样框的标志 } if self._use_group_sampling: # False ret["group_ids"] = np.array([s["group_id"] for s in sampled]) else: ret["group_ids"] = np.arange(gt_boxes.shape[0], gt_boxes.shape[0] + len(sampled)) # 采样对象序号 else: ret = None return ret
def sample_all(self, root_path, gt_boxes, gt_names, num_point_features, random_crop=False, gt_group_ids=None, rect=None, Trv2c=None, P2=None): sampled_num_dict = {} sample_num_per_class = [] for class_name, max_sample_num in zip(self._sample_classes, self._sample_max_nums): sampled_num = int(max_sample_num - np.sum([n == class_name for n in gt_names])) sampled_num = np.round(self._rate * sampled_num).astype(np.int64) sampled_num_dict[class_name] = sampled_num sample_num_per_class.append(sampled_num) sampled_groups = self._sample_classes if self._use_group_sampling: assert gt_group_ids is not None sampled_groups = [] sample_num_per_class = [] for group_name, class_names in self._group_name_to_names: sampled_nums_group = [sampled_num_dict[n] for n in class_names] sampled_num = np.max(sampled_nums_group) sample_num_per_class.append(sampled_num) sampled_groups.append(group_name) total_group_ids = gt_group_ids sampled = [] sampled_gt_boxes = [] avoid_coll_boxes = gt_boxes for class_name, sampled_num in zip(sampled_groups, sample_num_per_class): if sampled_num > 0: if self._use_group_sampling: sampled_cls = self.sample_group(class_name, sampled_num, avoid_coll_boxes, total_group_ids) else: sampled_cls = self.sample_class_v2(class_name, sampled_num, avoid_coll_boxes) sampled += sampled_cls if len(sampled_cls) > 0: if len(sampled_cls) == 1: sampled_gt_box = sampled_cls[0]["box3d_lidar"][ np.newaxis, ...] else: sampled_gt_box = np.stack( [s["box3d_lidar"] for s in sampled_cls], axis=0) sampled_gt_boxes += [sampled_gt_box] avoid_coll_boxes = np.concatenate( [avoid_coll_boxes, sampled_gt_box], axis=0) if self._use_group_sampling: if len(sampled_cls) == 1: sampled_group_ids = np.array( sampled_cls[0]["group_id"])[np.newaxis, ...] else: sampled_group_ids = np.stack( [s["group_id"] for s in sampled_cls], axis=0) total_group_ids = np.concatenate( [total_group_ids, sampled_group_ids], axis=0) if len(sampled) > 0: sampled_gt_boxes = np.concatenate(sampled_gt_boxes, axis=0) num_sampled = len(sampled) s_points_list = [] for info in sampled: s_points = np.fromfile(str( pathlib.Path(root_path) / info["path"]), dtype=np.float32) s_points = s_points.reshape([-1, num_point_features]) # if not add_rgb_to_points: # s_points = s_points[:, :4] if "rot_transform" in info: rot = info["rot_transform"] s_points[:, :3] = box_np_ops.rotation_points_single_angle( s_points[:, :3], rot, axis=2) s_points[:, :3] += info["box3d_lidar"][:3] s_points_list.append(s_points) # print(pathlib.Path(info["path"]).stem) # gt_bboxes = np.stack([s["bbox"] for s in sampled], axis=0) # if np.random.choice([False, True], replace=False, p=[0.3, 0.7]): # do random crop. if random_crop: s_points_list_new = [] gt_bboxes = box_np_ops.box3d_to_bbox(sampled_gt_boxes, rect, Trv2c, P2) crop_frustums = prep.random_crop_frustum( gt_bboxes, rect, Trv2c, P2) for i in range(crop_frustums.shape[0]): s_points = s_points_list[i] mask = prep.mask_points_in_corners( s_points, crop_frustums[i:i + 1]).reshape(-1) num_remove = np.sum(mask) if num_remove > 0 and (s_points.shape[0] - num_remove) > 15: s_points = s_points[np.logical_not(mask)] s_points_list_new.append(s_points) s_points_list = s_points_list_new ret = { "gt_names": np.array([s["name"] for s in sampled]), "difficulty": np.array([s["difficulty"] for s in sampled]), "gt_boxes": sampled_gt_boxes, "points": np.concatenate(s_points_list, axis=0), "gt_masks": np.ones((num_sampled, ), dtype=np.bool_) } if self._use_group_sampling: ret["group_ids"] = np.array([s["group_id"] for s in sampled]) else: ret["group_ids"] = np.arange(gt_boxes.shape[0], gt_boxes.shape[0] + len(sampled)) else: ret = None return ret
def sample_all(self, points, voxel_size, point_cloud_range, voxel_grids, root_path, gt_boxes, gt_names, num_point_features, random_crop=False, gt_group_ids=None, rect=None, Trv2c=None, P2=None): sampled_num_dict = {} sample_num_per_class = [] for class_name, max_sample_num in zip(self._sample_classes, self._sample_max_nums): sampled_num = int(max_sample_num - np.sum([n == class_name for n in gt_names])) sampled_num = np.round(self._rate * sampled_num).astype(np.int64) sampled_num_dict[class_name] = sampled_num sample_num_per_class.append(sampled_num) sampled_groups = self._sample_classes if self._use_group_sampling: assert gt_group_ids is not None sampled_groups = [] sample_num_per_class = [] for group_name, class_names in self._group_name_to_names: sampled_nums_group = [sampled_num_dict[n] for n in class_names] sampled_num = np.max(sampled_nums_group) sample_num_per_class.append(sampled_num) sampled_groups.append(group_name) total_group_ids = gt_group_ids sampled = [] sampled_gt_boxes = [] avoid_coll_boxes = gt_boxes finetune_by_grd = False voxel_scale = 3 voxel_size_scaled = voxel_size * voxel_scale voxel_grids_scale = voxel_grids // voxel_scale if finetune_by_grd: grd_filter = ground_filter() grd_filter.filter(points) grd_gridmask = _points_to_gridmask_2d(grd_filter.ground_pc, voxel_size_scaled, point_cloud_range[:3], voxel_grids_scale) else: grd_gridmask = None for class_name, sampled_num in zip(sampled_groups, sample_num_per_class): if sampled_num > 0: if finetune_by_grd: assert self._use_group_sampling is not True all_samples = self._sampler_dict[class_name].sample( sampled_num * 15) all_samples = copy.deepcopy(all_samples) all_samples = _fine_sample_by_grd(all_samples, grd_gridmask, voxel_size_scaled, point_cloud_range[:3], voxel_grids_scale) if len(all_samples) > sampled_num: #print(len(all_samples), '>', sampled_num) all_samples = all_samples[:sampled_num] else: all_samples = self._sampler_dict[class_name].sample( sampled_num) all_samples = copy.deepcopy(all_samples) if self._use_group_sampling: sampled_cls = self.sample_group(class_name, sampled_num, avoid_coll_boxes, total_group_ids) else: if len(all_samples) > 0: sampled_cls = self.sample_class_v2( all_samples, avoid_coll_boxes) else: sampled_cls = [] sampled += sampled_cls if len(sampled_cls) > 0: if len(sampled_cls) == 1: sampled_gt_box = sampled_cls[0]["box3d_lidar"][ np.newaxis, ...] else: sampled_gt_box = np.stack( [s["box3d_lidar"] for s in sampled_cls], axis=0) sampled_gt_boxes += [sampled_gt_box] avoid_coll_boxes = np.concatenate( [avoid_coll_boxes, sampled_gt_box], axis=0) if self._use_group_sampling: if len(sampled_cls) == 1: sampled_group_ids = np.array( sampled_cls[0]["group_id"])[np.newaxis, ...] else: sampled_group_ids = np.stack( [s["group_id"] for s in sampled_cls], axis=0) total_group_ids = np.concatenate( [total_group_ids, sampled_group_ids], axis=0) if len(sampled) > 0: sampled_gt_boxes = np.concatenate(sampled_gt_boxes, axis=0) num_sampled = len(sampled) s_points_list = [] for info in sampled: s_points = np.fromfile(str( pathlib.Path(root_path) / info["path"]), dtype=np.float32) s_points = s_points.reshape([-1, num_point_features]) # if not add_rgb_to_points: # s_points = s_points[:, :4] if "rot_transform" in info: rot = info["rot_transform"] s_points[:, :3] = box_np_ops.rotation_points_single_angle( s_points[:, :3], rot, axis=2) s_points[:, :3] += info["box3d_lidar"][:3] s_points_list.append(s_points) # print(pathlib.Path(info["path"]).stem) # gt_bboxes = np.stack([s["bbox"] for s in sampled], axis=0) # if np.random.choice([False, True], replace=False, p=[0.3, 0.7]): # do random crop. if random_crop: s_points_list_new = [] gt_bboxes = box_np_ops.box3d_to_bbox(sampled_gt_boxes, rect, Trv2c, P2) crop_frustums = prep.random_crop_frustum( gt_bboxes, rect, Trv2c, P2) for i in range(crop_frustums.shape[0]): s_points = s_points_list[i] mask = prep.mask_points_in_corners( s_points, crop_frustums[i:i + 1]).reshape(-1) num_remove = np.sum(mask) if num_remove > 0 and (s_points.shape[0] - num_remove) > 15: s_points = s_points[np.logical_not(mask)] s_points_list_new.append(s_points) s_points_list = s_points_list_new finetune_samples_axis_z = True if finetune_samples_axis_z: gt_box_bottom_avg = np.mean(gt_boxes[:, 2]) # finetune boxes sampled_gt_boxes_bottom = sampled_gt_boxes[:, 2] delta = sampled_gt_boxes_bottom - gt_box_bottom_avg sampled_gt_boxes[:, 2] -= delta # finetune points for i in range(len(s_points_list)): s_points_list[i][:, 2] -= delta[i] ret = { "gt_names": np.array([s["name"] for s in sampled]), "difficulty": np.array([s["difficulty"] for s in sampled]), "gt_boxes": sampled_gt_boxes, "points": np.concatenate(s_points_list, axis=0), "gt_masks": np.ones((num_sampled, ), dtype=np.bool_) } if self._use_group_sampling: ret["group_ids"] = np.array([s["group_id"] for s in sampled]) else: ret["group_ids"] = np.arange(gt_boxes.shape[0], gt_boxes.shape[0] + len(sampled)) else: ret = None return ret