def augment_fn(batch_xyz, batch_label): bsize, num_point, _ = batch_xyz.shape # shuffle the orders of samples in a batch idx = np.arange(bsize) np.random.shuffle(idx) batch_xyz = batch_xyz[idx, :, :] batch_label = batch_label[idx, :] # shuffle the point orders of each sample idx = np.arange(num_point) np.random.shuffle(idx) batch_xyz = batch_xyz[:, idx, :] batch_label = batch_label[:, idx] # perform augmentation on the first np.int32(augment_ratio*bsize) samples augSize = np.int32(1 / 3.0 * bsize) augment_xyz = batch_xyz[0:augSize] augment_xyz = data_util.rotate_point_cloud(augment_xyz) augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) augment_xyz = data_util.jitter_point_cloud(augment_xyz) batch_xyz[0:augSize] = augment_xyz augment_xyz = batch_xyz[augSize:2 * augSize] augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) augment_xyz = data_util.jitter_point_cloud(augment_xyz) batch_xyz[augSize:2 * augSize] = augment_xyz return batch_xyz, batch_label
def augment_fn(batch_input, batch_feature, augment_ratio=0.5): bsize, num_point, _ = batch_input.shape # shuffle the orders of samples in a batch idx = np.arange(bsize) np.random.shuffle(idx) batch_input = batch_input[idx, :, :] # batch_rho = batch_rho[idx,:] # batch_engy = batch_engy[idx,:] batch_feature = batch_feature[idx] # shuffle the point orders of each sample idx = np.arange(num_point) np.random.shuffle(idx) batch_input = batch_input[:, idx, :] # batch_rho = batch_rho[:,idx] # batch_engy = batch_engy[:,idx] # batch_xyz = data_util.shuffle_points(batch_xyz) # perform augmentation on the first np.int32(augment_ratio*bsize) samples augSize = np.int32(augment_ratio * bsize) augment_xyz = batch_input[0:augSize, :, 0:3] augment_xyz = data_util.rotate_point_cloud(augment_xyz) augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) batch_input[0:augSize, :, 0:3] = augment_xyz return batch_input, batch_feature
def augment_fn(batch_xyz): # perform augmentation on the first np.int32(augment_ratio*bsize) samples augment_xyz = data_util.rotate_point_cloud(batch_xyz) augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) return augment_xyz
def augment_fn(batch_input): augment_xyz = batch_input[:, :, 0:3] augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) augment_xyz = data_util.jitter_point_cloud(augment_xyz) batch_input[:, :, 0:3] = augment_xyz return batch_input
def augment_fn2(batch_xyz): augment_xyz = batch_xyz augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) augment_xyz = data_util.jitter_point_cloud(augment_xyz) batch_xyz = augment_xyz return batch_xyz
def augment_fn1(batch_input): # perform augmentation on the first np.int32(augment_ratio*bsize) samples augment_xyz = batch_input[:, :, 0:3] augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) augment_xyz = data_util.jitter_point_cloud(augment_xyz) batch_input[:, :, 0:3] = augment_xyz return batch_input
def augment_fn(batch_input, batch_label): bsize, num_point, _ = batch_input.shape # shuffle the orders of samples in a batch idx = np.arange(bsize) np.random.shuffle(idx) batch_input = batch_input[idx,:,:] batch_label = batch_label[idx,:] # shuffle the point orders of each sample idx = np.arange(num_point) np.random.shuffle(idx) batch_input = batch_input[:,idx,:] batch_label = batch_label[:,idx] # perform augmentation on the first np.int32(augment_ratio*bsize) samples augSize = np.int32(1/3.0 * bsize) if INPUT_DIM==6: augment_xyz = batch_input[0:augSize, :, 0:3] augment_xyz = data_util.rotate_point_cloud(augment_xyz) augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz) elif INPUT_DIM==9: augment_xyz = batch_input[0:augSize, :, 0:6] augment_xyz = data_util.rotate_point_cloud_with_normal(augment_xyz) augment_xyz = data_util.rotate_perturbation_point_cloud_with_normal(augment_xyz) augment_xyz[:,:,0:3] = data_util.random_scale_point_cloud(augment_xyz[:,:,0:3]) augment_xyz[:,:,0:3] = data_util.shift_point_cloud(augment_xyz[:,:,0:3]) augment_xyz[:,:,0:3] = data_util.jitter_point_cloud(augment_xyz[:,:,0:3]) if INPUT_DIM==6: batch_input[0:augSize, :, 0:3] = augment_xyz elif INPUT_DIM==9: batch_input[0:augSize, :, 0:6] = augment_xyz # one third of data without rotation augmentation augment_xyz = batch_input[augSize:2*augSize,:,0:3] augment_xyz = data_util.random_scale_point_cloud(augment_xyz) augment_xyz = data_util.shift_point_cloud(augment_xyz) augment_xyz = data_util.jitter_point_cloud(augment_xyz) batch_input[augSize:2*augSize,:,0:3] = augment_xyz return batch_input, batch_label
def load_frame_data(self, data_idx_str, random_flip=False, random_rotate=False, random_shift=False, pca_jitter=False): '''load one frame''' data_idx = int(data_idx_str) # print(data_idx_str) calib = self.kitti_dataset.get_calibration(data_idx) # 3 by 4 matrix if self.is_training: objects = self.kitti_dataset.get_label_objects(data_idx) else: objects = [] objects = filter( lambda obj: obj.type in self.types_list and obj.difficulty in self. difficulties_list, objects) gt_boxes = [] # ground truth boxes image = self.kitti_dataset.get_image(data_idx) pc_velo = self.kitti_dataset.get_lidar(data_idx) img_height, img_width = image.shape[0:2] # data augmentation if pca_jitter: image = apply_pca_jitter(image)[0] # if random_flip and np.random.random()>0.5: # 50% chance flipping if random_flip and False: pc_velo[:, 1] *= -1 # FIXME: to flip the scene, optical center also need to be adjusted # dont use with image now calib.P[0, 2] = img_width - calib.P[0, 2] image = np.flip(image, axis=1) for obj in objects: obj.t = [-obj.t[0], obj.t[1], obj.t[2]] # ensure that ry is [-pi, pi] if obj.ry >= 0: obj.ry = np.pi - obj.ry else: obj.ry = -np.pi - obj.ry # use original point cloud for rpn if not self.train_img_seg: _, pc_image_coord, img_fov_inds = get_lidar_in_image_fov( pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True) pc_velo = pc_velo[img_fov_inds, :] choice = np.random.choice(pc_velo.shape[0], self.npoints, replace=True) point_set = pc_velo[choice, :] pc_rect = np.zeros_like(point_set) pc_rect[:, 0:3] = calib.project_velo_to_rect(point_set[:, 0:3]) pc_rect[:, 3] = point_set[:, 3] else: # use dense point cloud for training image segmentation pc_dense = self.dense_points.load_dense_points(data_idx) # get_lidar_in_image_fov pts_2d = calib.project_rect_to_image(pc_dense) fov_inds = (pts_2d[:,0]<img_width) & (pts_2d[:,0]>=0) & \ (pts_2d[:,1]<img_height) & (pts_2d[:,1]>=0) # fov_inds = fov_inds & (pc_dense[:,2]>2.0) pc_dense = pc_dense[fov_inds, :] obj_mask = { 'Car': np.zeros((len(pc_dense), ), dtype=np.bool), 'Pedestrian': np.zeros((len(pc_dense), ), dtype=np.bool), 'Cyclist': np.zeros((len(pc_dense), ), dtype=np.bool) } for obj in objects: _, obj_box_3d = utils.compute_box_3d(obj, calib.P) _, mask = extract_pc_in_box3d(pc_dense, obj_box_3d) obj_mask[obj.type] = np.logical_or(mask, obj_mask[obj.type]) bg_mask = (obj_mask['Car'] + obj_mask['Pedestrian'] + obj_mask['Cyclist']) == 0 car = np.where(obj_mask['Car'])[0] # oversampling ped = np.where(obj_mask['Pedestrian'])[0] if ped.shape[0] > 0: #ped = ped[np.random.choice(ped.shape[0], car.shape[0], replace=True)] ped = ped[np.random.choice(ped.shape[0], ped.shape[0] * 5, replace=True)] cyc = np.where(obj_mask['Cyclist'])[0] if cyc.shape[0] > 0: #cyc = cyc[np.random.choice(cyc.shape[0], car.shape[0], replace=True)] cyc = cyc[np.random.choice(cyc.shape[0], cyc.shape[0] * 10, replace=True)] fg_num = car.shape[0] + ped.shape[0] + cyc.shape[0] #print('total points: ', len(pc_dense)) #print('fg_num: {0}, car: {1}, ped: {2}, cyc: {3}'.format(fg_num, car.shape[0], ped.shape[0], cyc.shape[0])) bg = np.where(bg_mask)[0] bg = bg[np.random.choice(bg.shape[0], max(self.npoints - fg_num, 0), replace=True)] choice = np.concatenate((car, ped, cyc, bg), axis=0)[:self.npoints] #choice = np.random.choice(pc_dense.shape[0], self.npoints, replace=True) pc_rect = np.zeros((self.npoints, 4)) pc_rect[:, :3] = pc_dense[choice, :] #start = time.time() seg_mask = np.zeros((pc_rect.shape[0])) # data augmentation # dont use with image now if random_rotate and False: ry = (np.random.random() - 0.5) * math.radians( 20) # -10~10 degrees pc_rect[:, 0:3] = rotate_points_along_y(pc_rect[:, 0:3], ry) for obj in objects: obj.t = rotate_points_along_y(obj.t, ry) obj.ry -= ry # ensure that ry is [-pi, pi] if obj.ry > np.pi: obj.ry -= 2 * np.pi elif obj.ry < -np.pi: obj.ry += 2 * np.pi proposal_of_point = {} # point index to proposal vector gt_box_of_point = {} # point index to corners_3d for obj in objects: _, obj_box_3d = utils.compute_box_3d(obj, calib.P) _, obj_mask = extract_pc_in_box3d(pc_rect, obj_box_3d) if np.sum(obj_mask) == 0: # label without 3d points # print('skip object without points') continue seg_mask[obj_mask] = g_type2onehotclass[obj.type] #seg_mask[obj_mask] = 1 gt_boxes.append(obj_box_3d) obj_idxs = np.where(obj_mask)[0] # data augmentation # FIXME: jitter point will make valid loss growing # Also may go out of image view if random_shift and False: # jitter object points pc_rect[obj_idxs, :3] = shift_point_cloud( pc_rect[obj_idxs, :3], 0.02) for idx in obj_idxs: # to save time if self.train_img_seg: proposal_of_point[idx] = [0, [0, 0, 0], 0, 0, 0, [0, 0, 0]] else: proposal_of_point[idx] = box_encoder.encode( obj, pc_rect[idx, :3]) gt_box_of_point[idx] = obj_box_3d # self.viz_frame(pc_rect, seg_mask, gt_boxes) # return pc_rect, seg_mask, proposal_of_point, gt_box_of_point, gt_boxes calib_matrix = np.copy(calib.P) calib_matrix[0, :] *= (1200.0 / image.shape[1]) calib_matrix[1, :] *= (360.0 / image.shape[0]) #print('construct', time.time() - start) return { 'pointcloud': pc_rect, 'image': cv2.resize(image, (1200, 360)), 'calib': calib_matrix, 'mask_label': seg_mask, 'proposal_of_point': self.get_proposal_out(proposal_of_point), 'gt_box_of_point': self.get_gt_box_of_points(gt_box_of_point), 'gt_boxes': gt_boxes, 'pc_choice': choice }
def load_frame_data(self, data_idx_str, random_flip=False, random_rotate=False, random_shift=False, pca_jitter=False): '''load one frame''' if self.use_aug_scene: data_idx = int(data_idx_str[2:]) else: data_idx = int(data_idx_str) # print(data_idx_str) calib = self.kitti_dataset.get_calibration(data_idx) # 3 by 4 matrix image = self.kitti_dataset.get_image(data_idx) img_height, img_width = image.shape[0:2] # data augmentation if pca_jitter: image = apply_pca_jitter(image)[0] objects = [] if not self.use_aug_scene or data_idx_str[:2] == '00': pc_velo = self.kitti_dataset.get_lidar(data_idx) _, pc_image_coord, img_fov_inds = get_lidar_in_image_fov( pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True) pc_velo = pc_velo[img_fov_inds, :] choice = np.random.choice(pc_velo.shape[0], self.npoints, replace=True) point_set = pc_velo[choice, :] pc_rect = np.zeros_like(point_set) pc_rect[:, 0:3] = calib.project_velo_to_rect(point_set[:, 0:3]) pc_rect[:, 3] = point_set[:, 3] if self.is_training: objects = self.kitti_dataset.get_label_objects(data_idx) else: pc_rect = utils.load_velo_scan( os.path.join( self.kitti_path, 'aug_scene/rectified_data/{0}.bin'.format(data_idx_str))) choice = np.random.choice(pc_rect.shape[0], self.npoints, replace=True) pc_rect = pc_rect[choice, :] if self.is_training: objects = utils.read_label( os.path.join( self.kitti_path, 'aug_scene/aug_label/{0}.txt'.format(data_idx_str))) objects = filter( lambda obj: obj.type in self.types_list and obj.difficulty in self. difficulties_list, objects) gt_boxes = [] # ground truth boxes #start = time.time() seg_mask = np.zeros((pc_rect.shape[0])) # data augmentation if random_flip and np.random.random() > 0.5: # 50% chance flipping pc_rect[:, 0] *= -1 for obj in objects: obj.t = [-obj.t[0], obj.t[1], obj.t[2]] # ensure that ry is [-pi, pi] if obj.ry >= 0: obj.ry = np.pi - obj.ry else: obj.ry = -np.pi - obj.ry if random_rotate: ry = (np.random.random() - 0.5) * math.radians( 20) # -10~10 degrees pc_rect[:, 0:3] = rotate_points_along_y(pc_rect[:, 0:3], ry) for obj in objects: obj.t = rotate_points_along_y(obj.t, ry) obj.ry -= ry # ensure that ry is [-pi, pi] if obj.ry > np.pi: obj.ry -= 2 * np.pi elif obj.ry < -np.pi: obj.ry += 2 * np.pi proposal_of_point = {} # point index to proposal vector gt_box_of_point = {} # point index to corners_3d for obj in objects: _, obj_box_3d = utils.compute_box_3d(obj, calib.P) _, obj_mask = extract_pc_in_box3d(pc_rect, obj_box_3d) if np.sum(obj_mask) == 0: # label without 3d points # print('skip object without points') continue # IMPORTANT: this must match with NUM_SEG_CLASSES #seg_mask[obj_mask] = g_type2onehotclass[obj.type] seg_mask[obj_mask] = 1 gt_boxes.append(obj_box_3d) obj_idxs = np.where(obj_mask)[0] # data augmentation # FIXME: jitter point will make valid loss growing # Also may go out of image view if random_shift and False: # jitter object points pc_rect[obj_idxs, :3] = shift_point_cloud( pc_rect[obj_idxs, :3], 0.02) for idx in obj_idxs: proposal_of_point[idx] = box_encoder.encode( obj, pc_rect[idx, :3]) gt_box_of_point[idx] = obj_box_3d # self.viz_frame(pc_rect, seg_mask, gt_boxes) # return pc_rect, seg_mask, proposal_of_point, gt_box_of_point, gt_boxes calib_matrix = np.copy(calib.P) calib_matrix[0, :] *= (1200.0 / image.shape[1]) calib_matrix[1, :] *= (360.0 / image.shape[0]) #print('construct', time.time() - start) return { 'pointcloud': pc_rect, 'image': cv2.resize(image, (1200, 360)), 'calib': calib_matrix, 'mask_label': seg_mask, 'proposal_of_point': self.get_proposal_out(proposal_of_point), 'gt_box_of_point': self.get_gt_box_of_points(gt_box_of_point), 'gt_boxes': gt_boxes, 'pc_choice': choice }