def visualize_2d_boxes_on_top_image(bboxes_grid,
top_view,
grid_meters,
bbox_voxel_size,
prediction=False):
top_image_vis = []
for boxes, top_image in zip(bboxes_grid, top_view): # iterate over batch
top_image = top_image.numpy()
shape = top_image.shape
rgb_image = np.zeros((shape[0], shape[1], 3))
rgb_image[top_image[:, :, 0] > 0] = 1
box, labels, _ = get_boxes_from_box_grid(boxes, bbox_voxel_size)
box = box.numpy()
box, orientation_3d = make_eight_points_boxes(box)
if prediction:
labels = np.argmax(labels, axis=-1)
if len(box) > 0:
rgb_image = draw_boxes_top_view(rgb_image, box, grid_meters,
labels, orientation_3d)
# rgb_image = np.rot90(rgb_image)
top_image_vis.append(rgb_image)
return np.asarray(top_image_vis)
def reproject_and_save(dataset_dir, inference_dir, camera_name, seq):
seq_number = seq.split("/")[-1]
bbox_list = sorted(glob.glob(seq + "/*.txt"))
image_f_dir = os.path.join(dataset_dir, seq_number, "camera", camera_name)
intrinsics_f = read_json(image_f_dir + "/intrinsics.json")
intr_f = intrinscs_to_matrix(**intrinsics_f)
poses_f = read_json(image_f_dir + "/poses.json")
lidar_poses = read_json(
os.path.join(dataset_dir, seq_number, "lidar", "poses.json"))
output_dir = os.path.join(inference_dir, "image_bboxes", seq_number,
camera_name)
os.makedirs(output_dir, exist_ok=True)
for idx, bbox_path in enumerate(bbox_list):
name = os.path.splitext(os.path.basename(bbox_path))[0]
cam_T_lidar = get_extrinsic(lidar_poses[idx], poses_f[idx])
# lidar_file = os.path.join(
# dataset_dir, seq_number, "lidar_processed", name + ".bin"
# )
# lidar = load_lidar(lidar_file)
# lidar = lidar[:, :3]
# Load boxes
bboxes = load_bboxes(bbox_path, label_string=False)
labels = bboxes[:, -1]
lidar_corners_3d, _ = make_eight_points_boxes(bboxes[:, :-1])
orient_3d = get_orient(lidar_corners_3d)
lidar_corners_3d = project_boxes_to_cam(lidar_corners_3d, cam_T_lidar)
orient_3d = project_boxes_to_cam(orient_3d, cam_T_lidar)
# lidar = project_lidar_to_cam(lidar, cam_T_lidar)
# figure = visualize_lidar(lidar)
# figure = visualize_bboxes_3d(lidar_corners_3d, figure)
# project to image
image_path = os.path.join(image_f_dir, name + ".jpg")
image_f = np.asarray(Image.open(image_path)) / 255
box_2d = project_to_image(lidar_corners_3d, intr_f)
orient_2d = project_to_image(orient_3d, intr_f)
image = visualize_bboxes_on_image(image_f, box_2d, labels,
orient_2d) * 255
image_name = os.path.join(output_dir, name + ".png")
img = Image.fromarray(image.astype("uint8"))
img.save(image_name)
def load_data(self, data_input):
"""
Loads image and semseg and resizes it
Note: This is numpy function.
"""
lidar_file, bboxes_file = np.asarray(data_input).astype("U")
lidar = load_lidar(lidar_file)
bboxes = load_bboxes(bboxes_file)
labels = bboxes[:, -1]
lidar_corners_3d, _ = make_eight_points_boxes(bboxes[:, :-1])
if self.augmentation:
np.random.shuffle(lidar)
if np.random.uniform(
0, 1) < 0.50: # 50% probability to flip over x axis
lidar, lidar_corners_3d = random_flip_x_lidar_boxes(
lidar, lidar_corners_3d)
if np.random.uniform(
0, 1) < 0.50: # 50% probability to flip over y axis
lidar, lidar_corners_3d = random_flip_y_lidar_boxes(
lidar, lidar_corners_3d)
if np.random.uniform(0, 1) < 0.80: # 80% probability to rotate
lidar, lidar_corners_3d = random_rotate_lidar_boxes(
lidar, lidar_corners_3d)
# # Shift lidar coordinate to positive quadrant
lidar_coord = np.asarray(self.param_settings["lidar_offset"],
dtype=np.float32)
lidar = lidar + lidar_coord
lidar_corners_3d = lidar_corners_3d + lidar_coord[:3]
# Process data
top_view = make_top_view_image(lidar,
self.param_settings["grid_meters"],
self.param_settings["voxel_size"])
box_grid = get_bboxes_grid(
labels,
lidar_corners_3d,
self.param_settings["grid_meters"],
self.param_settings["bbox_voxel_size"],
)
return top_view, box_grid, lidar_file
def preprocess_data(dataset_dir):
"""
The function visualizes data from pandaset.
Arguments:
dataset_dir: directory with Pandaset data
"""
shift_lidar = [25, 50, 2.5]
# Get list of data samples
search_string = os.path.join(dataset_dir, "*")
seq_list = sorted(glob.glob(search_string))
for seq in tqdm(seq_list, desc="Process sequences", total=len(seq_list)):
search_string = os.path.join(seq, "lidar", "*.pkl.gz")
lidar_list = sorted(glob.glob(search_string))
lidar_pose_path = os.path.join(seq, "lidar", "poses.json")
lidar_pose = read_json(lidar_pose_path)
for idx, lidar_path in enumerate(lidar_list):
# Get pose of the lidar
translation = lidar_pose[idx]["position"]
translation = np.asarray([translation[key] for key in translation])
rotation = lidar_pose[idx]["heading"]
rotation = np.asarray([rotation[key] for key in rotation])
rotation = quaternion_to_euler(*rotation)
Rt = to_transform_matrix(translation, rotation)
# Get respective bboxes
bbox_path = lidar_path.split("/")
bbox_path[-2] = "annotations/cuboids"
bbox_path = os.path.join(*bbox_path)
# Load data
lidar = np.asarray(pd.read_pickle(lidar_path))
# Get only lidar 0 (there is also lidar 1)
lidar = lidar[lidar[:, -1] == 0]
intensity = lidar[:, 3]
lidar = transform_lidar_box_3d(lidar, Rt)
# add intensity
lidar = np.concatenate((lidar, intensity[:, None]), axis=-1)
# Load bboxes
bboxes = np.asarray(pd.read_pickle(bbox_path))
labels, bboxes = make_xzyhwly(bboxes)
corners_3d, orientation_3d = make_eight_points_boxes(bboxes)
corners_3d = np.asarray(
[transform_lidar_box_3d(box, Rt) for box in corners_3d])
orientation_3d = np.asarray(
[transform_lidar_box_3d(box, Rt) for box in orientation_3d])
labels, corners_3d, orientation_3d = filter_boxes(
labels, corners_3d, orientation_3d, lidar)
centroid, width, length, height, yaw = get_bboxes_parameters_from_points(
corners_3d)
boxes_new = np.concatenate(
(
centroid,
length[:, None],
width[:, None],
height[:, None],
yaw[:, None],
),
axis=-1,
)
lidar[:, :3] = lidar[:, :3] + shift_lidar
corners_3d, orientation_3d = make_eight_points_boxes(boxes_new)
corners_3d = corners_3d + shift_lidar
orientation_3d = orientation_3d + shift_lidar
figure = visualize_bboxes_3d(corners_3d, None, orientation_3d)
figure = visualize_lidar(lidar, figure)
mlab.show(1)
input()
mlab.close(figure)
def preprocess_data(dataset_dir):
"""
The function visualizes data from pandaset.
Arguments:
dataset_dir: directory with Pandaset data
"""
# Get list of data samples
search_string = os.path.join(dataset_dir, "*")
seq_list = sorted(glob.glob(search_string))
for seq in tqdm(seq_list, desc="Process sequences", total=len(seq_list)):
# Make output dirs for data
lidar_out_dir = os.path.join(seq, "lidar_processed")
bbox_out_dir = os.path.join(seq, "bbox_processed")
os.makedirs(lidar_out_dir, exist_ok=True)
os.makedirs(bbox_out_dir, exist_ok=True)
search_string = os.path.join(seq, "lidar", "*.pkl.gz")
lidar_list = sorted(glob.glob(search_string))
lidar_pose_path = os.path.join(seq, "lidar", "poses.json")
lidar_pose = read_json(lidar_pose_path)
for idx, lidar_path in enumerate(lidar_list):
sample_idx = os.path.splitext(os.path.basename(lidar_path))[0].split(".")[0]
# Get pose of the lidar
translation = lidar_pose[idx]["position"]
translation = np.asarray([translation[key] for key in translation])
rotation = lidar_pose[idx]["heading"]
rotation = np.asarray([rotation[key] for key in rotation])
rotation = quaternion_to_euler(*rotation)
Rt = to_transform_matrix(translation, rotation)
# Get respective bboxes
bbox_path = lidar_path.split("/")
bbox_path[-2] = "annotations/cuboids"
bbox_path = os.path.join(*bbox_path)
# Load data
lidar = np.asarray(pd.read_pickle(lidar_path))
# Get only lidar 0 (there is also lidar 1)
lidar = lidar[lidar[:, -1] == 0]
intensity = lidar[:, 3]
lidar = transform_lidar_box_3d(lidar, Rt)
# add intensity
lidar = np.concatenate((lidar, intensity[:, None]), axis=-1)
# Load bboxes
bboxes = np.asarray(pd.read_pickle(bbox_path))
labels, bboxes = make_xzyhwly(bboxes)
corners_3d, orientation_3d = make_eight_points_boxes(bboxes)
corners_3d = np.asarray(
[transform_lidar_box_3d(box, Rt) for box in corners_3d]
)
orientation_3d = np.asarray(
[transform_lidar_box_3d(box, Rt) for box in orientation_3d]
)
# filter boxes containing less then 20 lidar points inside
labels, corners_3d, orientation_3d = filter_boxes(
labels, corners_3d, orientation_3d, lidar
)
centroid, width, length, height, yaw = get_bboxes_parameters_from_points(
corners_3d
)
# Save data
lidar_filename = os.path.join(lidar_out_dir, sample_idx + ".bin")
save_lidar(lidar_filename, lidar.astype(np.float32))
box_filename = os.path.join(bbox_out_dir, sample_idx + ".txt")
save_bboxes_to_file(
box_filename, centroid, width, length, height, yaw, labels
)
cam_T_lid = get_extrinsic(pose_l[0], pose_c[0])
lidar = load_lidar(lidar_file)
lidar = lidar[:, :3]
lidar = np.transpose(lidar)
ones = np.ones_like(lidar[0])
lidar_hom = np.concatenate((lidar, ones[None, :]), axis=0)
lidar_cam = np.matmul(cam_T_lid, lidar_hom)
lidar_cam = np.transpose(lidar_cam)
lidar = lidar_cam[:, :3]
lidar = lidar[lidar[:, 2] >= 0]
bboxes = load_bboxes(bboxes_file)
labels = bboxes[:, -1]
print(f"bboxes[:, 1:] {bboxes.shape}")
lidar_corners_3d, orient = make_eight_points_boxes(bboxes[:, :-1])
(
centroid,
width,
length,
height,
yaw,
) = get_bboxes_parameters_from_points(lidar_corners_3d)
print(f"shape {centroid.shape}, height {height.shape} yaw {yaw.shape}")
box_xyzlwhy = np.concatenate(
(centroid, length[:, None], width[:, None], height[:, None], yaw[:, None]),
axis=-1)
lidar_corners_3d, orient = make_eight_points_boxes(box_xyzlwhy)
# # Shift lidar coordinate to positive quadrant
lidar_coord = np.asarray(param_settings["lidar_offset"], dtype=np.float32)
def validation_inference(param_settings, dataset_file, model_dir, output_dir):
setup_gpu()
# Load model
model = tf.keras.models.load_model(model_dir)
bbox_voxel_size = np.asarray(param_settings["bbox_voxel_size"],
dtype=np.float32)
lidar_coord = np.array(param_settings["lidar_offset"], dtype=np.float32)
grid_meters = param_settings["grid_meters"]
val_dataset = DetectionDataset(param_settings, dataset_file, shuffle=False)
param_settings["val_size"] = val_dataset.num_samples
for val_samples in tqdm(
val_dataset.dataset,
desc=f"val_inference",
total=val_dataset.num_it_per_epoch,
):
top_view, gt_boxes, lidar_filenames = val_samples
predictions = model(top_view, training=False)
for image, predict, gt, filename in zip(top_view, predictions,
gt_boxes, lidar_filenames):
filename = str(filename.numpy())
seq_folder = filename.split("/")[-3]
name = os.path.splitext(os.path.basename(filename))[0]
# Ensure that output dir exists or create it
top_view_dir = os.path.join(output_dir, "top_view", seq_folder)
bboxes_dir = os.path.join(output_dir, "bboxes", seq_folder)
os.makedirs(top_view_dir, exist_ok=True)
os.makedirs(bboxes_dir, exist_ok=True)
p_top_view = (visualize_2d_boxes_on_top_image(
[predict],
[image],
grid_meters,
bbox_voxel_size,
prediction=True,
) * 255)
gt_top_view = (visualize_2d_boxes_on_top_image(
[gt],
[image],
grid_meters,
bbox_voxel_size,
prediction=False,
) * 255)
result = np.vstack((p_top_view[0], gt_top_view[0]))
file_to_save = os.path.join(top_view_dir, name + ".png")
img = Image.fromarray(result.astype("uint8"))
img.save(file_to_save)
box, labels, _ = get_boxes_from_box_grid(predict, bbox_voxel_size)
box = box.numpy()
box, _ = make_eight_points_boxes(box)
if len(box) > 0:
box = box - lidar_coord[:3]
labels = np.argmax(labels, axis=-1)
(
centroid,
width,
length,
height,
yaw,
) = get_bboxes_parameters_from_points(box)
bboxes_name = os.path.join(bboxes_dir, name + ".txt")
save_bboxes_to_file(bboxes_name, centroid, width, length,
height, yaw, labels)