def augment_fn(batch_xyz, batch_label):
bsize, num_point, _ = batch_xyz.shape
# shuffle the orders of samples in a batch
idx = np.arange(bsize)
np.random.shuffle(idx)
batch_xyz = batch_xyz[idx, :, :]
batch_label = batch_label[idx, :]
# shuffle the point orders of each sample
idx = np.arange(num_point)
np.random.shuffle(idx)
batch_xyz = batch_xyz[:, idx, :]
batch_label = batch_label[:, idx]
# perform augmentation on the first np.int32(augment_ratio*bsize) samples
augSize = np.int32(1 / 3.0 * bsize)
augment_xyz = batch_xyz[0:augSize]
augment_xyz = data_util.rotate_point_cloud(augment_xyz)
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
augment_xyz = data_util.jitter_point_cloud(augment_xyz)
batch_xyz[0:augSize] = augment_xyz
augment_xyz = batch_xyz[augSize:2 * augSize]
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
augment_xyz = data_util.jitter_point_cloud(augment_xyz)
batch_xyz[augSize:2 * augSize] = augment_xyz
return batch_xyz, batch_label
def augment_fn(batch_input, batch_feature, augment_ratio=0.5):
bsize, num_point, _ = batch_input.shape
# shuffle the orders of samples in a batch
idx = np.arange(bsize)
np.random.shuffle(idx)
batch_input = batch_input[idx, :, :]
# batch_rho = batch_rho[idx,:]
# batch_engy = batch_engy[idx,:]
batch_feature = batch_feature[idx]
# shuffle the point orders of each sample
idx = np.arange(num_point)
np.random.shuffle(idx)
batch_input = batch_input[:, idx, :]
# batch_rho = batch_rho[:,idx]
# batch_engy = batch_engy[:,idx]
# batch_xyz = data_util.shuffle_points(batch_xyz)
# perform augmentation on the first np.int32(augment_ratio*bsize) samples
augSize = np.int32(augment_ratio * bsize)
augment_xyz = batch_input[0:augSize, :, 0:3]
augment_xyz = data_util.rotate_point_cloud(augment_xyz)
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
batch_input[0:augSize, :, 0:3] = augment_xyz
return batch_input, batch_feature
def augment_fn(batch_xyz):
# perform augmentation on the first np.int32(augment_ratio*bsize) samples
augment_xyz = data_util.rotate_point_cloud(batch_xyz)
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
return augment_xyz
def augment_fn(batch_input):
augment_xyz = batch_input[:, :, 0:3]
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
augment_xyz = data_util.jitter_point_cloud(augment_xyz)
batch_input[:, :, 0:3] = augment_xyz
return batch_input
def augment_fn2(batch_xyz):
augment_xyz = batch_xyz
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
augment_xyz = data_util.jitter_point_cloud(augment_xyz)
batch_xyz = augment_xyz
return batch_xyz
def augment_fn1(batch_input):
# perform augmentation on the first np.int32(augment_ratio*bsize) samples
augment_xyz = batch_input[:, :, 0:3]
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
augment_xyz = data_util.jitter_point_cloud(augment_xyz)
batch_input[:, :, 0:3] = augment_xyz
return batch_input
def augment_fn(batch_input, batch_label):
bsize, num_point, _ = batch_input.shape
# shuffle the orders of samples in a batch
idx = np.arange(bsize)
np.random.shuffle(idx)
batch_input = batch_input[idx,:,:]
batch_label = batch_label[idx,:]
# shuffle the point orders of each sample
idx = np.arange(num_point)
np.random.shuffle(idx)
batch_input = batch_input[:,idx,:]
batch_label = batch_label[:,idx]
# perform augmentation on the first np.int32(augment_ratio*bsize) samples
augSize = np.int32(1/3.0 * bsize)
if INPUT_DIM==6:
augment_xyz = batch_input[0:augSize, :, 0:3]
augment_xyz = data_util.rotate_point_cloud(augment_xyz)
augment_xyz = data_util.rotate_perturbation_point_cloud(augment_xyz)
elif INPUT_DIM==9:
augment_xyz = batch_input[0:augSize, :, 0:6]
augment_xyz = data_util.rotate_point_cloud_with_normal(augment_xyz)
augment_xyz = data_util.rotate_perturbation_point_cloud_with_normal(augment_xyz)
augment_xyz[:,:,0:3] = data_util.random_scale_point_cloud(augment_xyz[:,:,0:3])
augment_xyz[:,:,0:3] = data_util.shift_point_cloud(augment_xyz[:,:,0:3])
augment_xyz[:,:,0:3] = data_util.jitter_point_cloud(augment_xyz[:,:,0:3])
if INPUT_DIM==6:
batch_input[0:augSize, :, 0:3] = augment_xyz
elif INPUT_DIM==9:
batch_input[0:augSize, :, 0:6] = augment_xyz
# one third of data without rotation augmentation
augment_xyz = batch_input[augSize:2*augSize,:,0:3]
augment_xyz = data_util.random_scale_point_cloud(augment_xyz)
augment_xyz = data_util.shift_point_cloud(augment_xyz)
augment_xyz = data_util.jitter_point_cloud(augment_xyz)
batch_input[augSize:2*augSize,:,0:3] = augment_xyz
return batch_input, batch_label
def load_frame_data(self,
data_idx_str,
random_flip=False,
random_rotate=False,
random_shift=False,
pca_jitter=False):
'''load one frame'''
data_idx = int(data_idx_str)
# print(data_idx_str)
calib = self.kitti_dataset.get_calibration(data_idx) # 3 by 4 matrix
if self.is_training:
objects = self.kitti_dataset.get_label_objects(data_idx)
else:
objects = []
objects = filter(
lambda obj: obj.type in self.types_list and obj.difficulty in self.
difficulties_list, objects)
gt_boxes = [] # ground truth boxes
image = self.kitti_dataset.get_image(data_idx)
pc_velo = self.kitti_dataset.get_lidar(data_idx)
img_height, img_width = image.shape[0:2]
# data augmentation
if pca_jitter:
image = apply_pca_jitter(image)[0]
# if random_flip and np.random.random()>0.5: # 50% chance flipping
if random_flip and False:
pc_velo[:, 1] *= -1
# FIXME: to flip the scene, optical center also need to be adjusted
# dont use with image now
calib.P[0, 2] = img_width - calib.P[0, 2]
image = np.flip(image, axis=1)
for obj in objects:
obj.t = [-obj.t[0], obj.t[1], obj.t[2]]
# ensure that ry is [-pi, pi]
if obj.ry >= 0:
obj.ry = np.pi - obj.ry
else:
obj.ry = -np.pi - obj.ry
# use original point cloud for rpn
if not self.train_img_seg:
_, pc_image_coord, img_fov_inds = get_lidar_in_image_fov(
pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True)
pc_velo = pc_velo[img_fov_inds, :]
choice = np.random.choice(pc_velo.shape[0],
self.npoints,
replace=True)
point_set = pc_velo[choice, :]
pc_rect = np.zeros_like(point_set)
pc_rect[:, 0:3] = calib.project_velo_to_rect(point_set[:, 0:3])
pc_rect[:, 3] = point_set[:, 3]
else:
# use dense point cloud for training image segmentation
pc_dense = self.dense_points.load_dense_points(data_idx)
# get_lidar_in_image_fov
pts_2d = calib.project_rect_to_image(pc_dense)
fov_inds = (pts_2d[:,0]<img_width) & (pts_2d[:,0]>=0) & \
(pts_2d[:,1]<img_height) & (pts_2d[:,1]>=0)
# fov_inds = fov_inds & (pc_dense[:,2]>2.0)
pc_dense = pc_dense[fov_inds, :]
obj_mask = {
'Car': np.zeros((len(pc_dense), ), dtype=np.bool),
'Pedestrian': np.zeros((len(pc_dense), ), dtype=np.bool),
'Cyclist': np.zeros((len(pc_dense), ), dtype=np.bool)
}
for obj in objects:
_, obj_box_3d = utils.compute_box_3d(obj, calib.P)
_, mask = extract_pc_in_box3d(pc_dense, obj_box_3d)
obj_mask[obj.type] = np.logical_or(mask, obj_mask[obj.type])
bg_mask = (obj_mask['Car'] + obj_mask['Pedestrian'] +
obj_mask['Cyclist']) == 0
car = np.where(obj_mask['Car'])[0]
# oversampling
ped = np.where(obj_mask['Pedestrian'])[0]
if ped.shape[0] > 0:
#ped = ped[np.random.choice(ped.shape[0], car.shape[0], replace=True)]
ped = ped[np.random.choice(ped.shape[0],
ped.shape[0] * 5,
replace=True)]
cyc = np.where(obj_mask['Cyclist'])[0]
if cyc.shape[0] > 0:
#cyc = cyc[np.random.choice(cyc.shape[0], car.shape[0], replace=True)]
cyc = cyc[np.random.choice(cyc.shape[0],
cyc.shape[0] * 10,
replace=True)]
fg_num = car.shape[0] + ped.shape[0] + cyc.shape[0]
#print('total points: ', len(pc_dense))
#print('fg_num: {0}, car: {1}, ped: {2}, cyc: {3}'.format(fg_num, car.shape[0], ped.shape[0], cyc.shape[0]))
bg = np.where(bg_mask)[0]
bg = bg[np.random.choice(bg.shape[0],
max(self.npoints - fg_num, 0),
replace=True)]
choice = np.concatenate((car, ped, cyc, bg), axis=0)[:self.npoints]
#choice = np.random.choice(pc_dense.shape[0], self.npoints, replace=True)
pc_rect = np.zeros((self.npoints, 4))
pc_rect[:, :3] = pc_dense[choice, :]
#start = time.time()
seg_mask = np.zeros((pc_rect.shape[0]))
# data augmentation
# dont use with image now
if random_rotate and False:
ry = (np.random.random() - 0.5) * math.radians(
20) # -10~10 degrees
pc_rect[:, 0:3] = rotate_points_along_y(pc_rect[:, 0:3], ry)
for obj in objects:
obj.t = rotate_points_along_y(obj.t, ry)
obj.ry -= ry
# ensure that ry is [-pi, pi]
if obj.ry > np.pi:
obj.ry -= 2 * np.pi
elif obj.ry < -np.pi:
obj.ry += 2 * np.pi
proposal_of_point = {} # point index to proposal vector
gt_box_of_point = {} # point index to corners_3d
for obj in objects:
_, obj_box_3d = utils.compute_box_3d(obj, calib.P)
_, obj_mask = extract_pc_in_box3d(pc_rect, obj_box_3d)
if np.sum(obj_mask) == 0:
# label without 3d points
# print('skip object without points')
continue
seg_mask[obj_mask] = g_type2onehotclass[obj.type]
#seg_mask[obj_mask] = 1
gt_boxes.append(obj_box_3d)
obj_idxs = np.where(obj_mask)[0]
# data augmentation
# FIXME: jitter point will make valid loss growing
# Also may go out of image view
if random_shift and False: # jitter object points
pc_rect[obj_idxs, :3] = shift_point_cloud(
pc_rect[obj_idxs, :3], 0.02)
for idx in obj_idxs:
# to save time
if self.train_img_seg:
proposal_of_point[idx] = [0, [0, 0, 0], 0, 0, 0, [0, 0, 0]]
else:
proposal_of_point[idx] = box_encoder.encode(
obj, pc_rect[idx, :3])
gt_box_of_point[idx] = obj_box_3d
# self.viz_frame(pc_rect, seg_mask, gt_boxes)
# return pc_rect, seg_mask, proposal_of_point, gt_box_of_point, gt_boxes
calib_matrix = np.copy(calib.P)
calib_matrix[0, :] *= (1200.0 / image.shape[1])
calib_matrix[1, :] *= (360.0 / image.shape[0])
#print('construct', time.time() - start)
return {
'pointcloud': pc_rect,
'image': cv2.resize(image, (1200, 360)),
'calib': calib_matrix,
'mask_label': seg_mask,
'proposal_of_point': self.get_proposal_out(proposal_of_point),
'gt_box_of_point': self.get_gt_box_of_points(gt_box_of_point),
'gt_boxes': gt_boxes,
'pc_choice': choice
}
def load_frame_data(self,
data_idx_str,
random_flip=False,
random_rotate=False,
random_shift=False,
pca_jitter=False):
'''load one frame'''
if self.use_aug_scene:
data_idx = int(data_idx_str[2:])
else:
data_idx = int(data_idx_str)
# print(data_idx_str)
calib = self.kitti_dataset.get_calibration(data_idx) # 3 by 4 matrix
image = self.kitti_dataset.get_image(data_idx)
img_height, img_width = image.shape[0:2]
# data augmentation
if pca_jitter:
image = apply_pca_jitter(image)[0]
objects = []
if not self.use_aug_scene or data_idx_str[:2] == '00':
pc_velo = self.kitti_dataset.get_lidar(data_idx)
_, pc_image_coord, img_fov_inds = get_lidar_in_image_fov(
pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True)
pc_velo = pc_velo[img_fov_inds, :]
choice = np.random.choice(pc_velo.shape[0],
self.npoints,
replace=True)
point_set = pc_velo[choice, :]
pc_rect = np.zeros_like(point_set)
pc_rect[:, 0:3] = calib.project_velo_to_rect(point_set[:, 0:3])
pc_rect[:, 3] = point_set[:, 3]
if self.is_training:
objects = self.kitti_dataset.get_label_objects(data_idx)
else:
pc_rect = utils.load_velo_scan(
os.path.join(
self.kitti_path,
'aug_scene/rectified_data/{0}.bin'.format(data_idx_str)))
choice = np.random.choice(pc_rect.shape[0],
self.npoints,
replace=True)
pc_rect = pc_rect[choice, :]
if self.is_training:
objects = utils.read_label(
os.path.join(
self.kitti_path,
'aug_scene/aug_label/{0}.txt'.format(data_idx_str)))
objects = filter(
lambda obj: obj.type in self.types_list and obj.difficulty in self.
difficulties_list, objects)
gt_boxes = [] # ground truth boxes
#start = time.time()
seg_mask = np.zeros((pc_rect.shape[0]))
# data augmentation
if random_flip and np.random.random() > 0.5: # 50% chance flipping
pc_rect[:, 0] *= -1
for obj in objects:
obj.t = [-obj.t[0], obj.t[1], obj.t[2]]
# ensure that ry is [-pi, pi]
if obj.ry >= 0:
obj.ry = np.pi - obj.ry
else:
obj.ry = -np.pi - obj.ry
if random_rotate:
ry = (np.random.random() - 0.5) * math.radians(
20) # -10~10 degrees
pc_rect[:, 0:3] = rotate_points_along_y(pc_rect[:, 0:3], ry)
for obj in objects:
obj.t = rotate_points_along_y(obj.t, ry)
obj.ry -= ry
# ensure that ry is [-pi, pi]
if obj.ry > np.pi:
obj.ry -= 2 * np.pi
elif obj.ry < -np.pi:
obj.ry += 2 * np.pi
proposal_of_point = {} # point index to proposal vector
gt_box_of_point = {} # point index to corners_3d
for obj in objects:
_, obj_box_3d = utils.compute_box_3d(obj, calib.P)
_, obj_mask = extract_pc_in_box3d(pc_rect, obj_box_3d)
if np.sum(obj_mask) == 0:
# label without 3d points
# print('skip object without points')
continue
# IMPORTANT: this must match with NUM_SEG_CLASSES
#seg_mask[obj_mask] = g_type2onehotclass[obj.type]
seg_mask[obj_mask] = 1
gt_boxes.append(obj_box_3d)
obj_idxs = np.where(obj_mask)[0]
# data augmentation
# FIXME: jitter point will make valid loss growing
# Also may go out of image view
if random_shift and False: # jitter object points
pc_rect[obj_idxs, :3] = shift_point_cloud(
pc_rect[obj_idxs, :3], 0.02)
for idx in obj_idxs:
proposal_of_point[idx] = box_encoder.encode(
obj, pc_rect[idx, :3])
gt_box_of_point[idx] = obj_box_3d
# self.viz_frame(pc_rect, seg_mask, gt_boxes)
# return pc_rect, seg_mask, proposal_of_point, gt_box_of_point, gt_boxes
calib_matrix = np.copy(calib.P)
calib_matrix[0, :] *= (1200.0 / image.shape[1])
calib_matrix[1, :] *= (360.0 / image.shape[0])
#print('construct', time.time() - start)
return {
'pointcloud': pc_rect,
'image': cv2.resize(image, (1200, 360)),
'calib': calib_matrix,
'mask_label': seg_mask,
'proposal_of_point': self.get_proposal_out(proposal_of_point),
'gt_box_of_point': self.get_gt_box_of_points(gt_box_of_point),
'gt_boxes': gt_boxes,
'pc_choice': choice
}
