def get_depth(self, folder, frame_index, side, do_flip):
"""Cropping will affect depth_gt and P_rect_norm (normalized intrinsic matrix)
"""
velo_filename = os.path.join(
self.data_path, folder,
"velodyne/{:010d}.bin".format(int(frame_index)))
calib_filename = os.path.join(
self.data_path, folder,
"calib/{:010d}.txt".format(int(frame_index)))
velo_rect, cam_intr = generate_depth_map_lyft(calib_filename,
velo_filename,
self.side_map[side])
### handle cropping
cam_intr[0, 2] -= self.crop_cols[0]
cam_intr[1, 2] -= self.crop_rows[0]
P_rect_norm = np.identity(4)
P_rect_norm[0, :] = cam_intr[0, :] / float(self.full_res_shape[0])
P_rect_norm[1, :] = cam_intr[1, :] / float(self.full_res_shape[1])
depth_gt = project_lidar_to_img(velo_rect,
P_rect_norm,
self.full_res_shape[::-1],
lyft_mode=True)
if do_flip:
depth_gt = np.fliplr(depth_gt)
return depth_gt, velo_rect, P_rect_norm
def export_gt_depths_kitti():
parser = argparse.ArgumentParser(description='export_gt_depth')
parser.add_argument('--data_path',
type=str,
help='path to the root of the KITTI data',
required=True)
parser.add_argument('--split',
type=str,
help='which split to export gt from',
required=True,
choices=["eigen", "eigen_benchmark"])
opt = parser.parse_args()
split_folder = os.path.join(os.path.dirname(__file__), "splits", opt.split)
lines = readlines(os.path.join(split_folder, "test_files.txt"))
print("Exporting ground truth depths for {}".format(opt.split))
gt_depths = []
for line in lines:
folder, frame_id, _ = line.split()
frame_id = int(frame_id)
if opt.split == "eigen":
calib_dir = os.path.join(opt.data_path, folder.split("/")[0])
velo_filename = os.path.join(opt.data_path, folder,
"velodyne_points/data", "{:010d}.bin".format(frame_id))
# gt_depth = generate_depth_map(calib_dir, velo_filename, 2, True) ## ZMH: This won't work because the generate_depth_map function has been redefined.
# gt_depth = generate_depth_map_original(calib_dir, velo_filename, 2, True) ## ZMH: the original function in monodepth2
# gt_depth = generate_depth_map_original(calib_dir, velo_filename, 2, False) ## ZMH: the original function in monodepth2, use transformed depth
velo_rect, P_rect_norm, im_shape = generate_depth_map(calib_dir, velo_filename, 2)
gt_depth = project_lidar_to_img(velo_rect, P_rect_norm, im_shape) ## ZMH: the way gt is generated I used in training
elif opt.split == "eigen_benchmark":
# gt_depth_path = os.path.join(opt.data_path, folder, "proj_depth",
# "groundtruth", "image_02", "{:010d}.png".format(frame_id))
gt_depth_path = os.path.join(opt.data_path, folder, "proj_depth",
"groundtruth", "image_02", "{:010d}.png".format(frame_id), 'val', folder.split("/")[1], "proj_depth",
"groundtruth", "image_02", "{:010d}.png".format(frame_id))
if not os.path.exists(gt_depth_path):
gt_depth_path = os.path.join(opt.data_path, folder, "proj_depth",
"groundtruth", "image_02", "{:010d}.png".format(frame_id), 'train', folder.split("/")[1], "proj_depth",
"groundtruth", "image_02", "{:010d}.png".format(frame_id))
if not os.path.exists(gt_depth_path):
raise ValueError("This file does not exist! {} {}".format(folder, frame_id))
gt_depth = np.array(pil.open(gt_depth_path)).astype(np.float32) / 256
gt_depths.append(gt_depth.astype(np.float32))
output_path = os.path.join(split_folder, "gt_depths_im_cus.npz")
print("Saving to {}".format(opt.split))
np.savez_compressed(output_path, data=np.array(gt_depths))
def get_depth(self, folder, frame_index, side, do_flip):
calib_path = os.path.join(self.data_path, folder.split("/")[0])
velo_filename = os.path.join(
self.data_path, folder,
"velodyne_points/data/{:010d}.bin".format(int(frame_index)))
velo_rect, P_rect_norm, im_shape = generate_depth_map(
calib_path, velo_filename, self.side_map[side])
depth_gt = project_lidar_to_img(velo_rect, P_rect_norm,
self.full_res_shape[::-1])
### ZMH: changed by me.
### The shape is just a little bit different, not huge resizing, because the depth_gt before resizing have a little bit different sizes
# depth_gt = skimage.transform.resize(
# depth_gt, self.full_res_shape[::-1], order=0, preserve_range=True, mode='constant') # don't use this one
# depth_gt = skimage.transform.resize(
# depth_gt, self.full_res_shape[::-1], order=0, preserve_range=True, mode='constant', anti_aliasing=False)
if do_flip:
depth_gt = np.fliplr(depth_gt)
# velo_rect = flip_lidar(velo_rect, P_rect_norm) # ZMH: add by me
return depth_gt, velo_rect, P_rect_norm
def get_depth_related(self, folder, frame_index, side, do_flip, inputs):
depth_gt, _, K_ = self.get_depth(folder, frame_index, side, do_flip)
inputs["depth_gt"] = np.expand_dims(depth_gt, 0)
inputs["depth_gt"] = torch.from_numpy(inputs["depth_gt"].astype(
np.float32))
## ZMH: load intrinsic matrix K here
for scale in range(self.num_scales):
K = K_.copy()
K[0, :] *= self.width // (2**scale)
K[1, :] *= self.height // (2**scale)
inv_K = np.linalg.pinv(K)
inputs[("K", scale)] = torch.from_numpy(K).to(dtype=torch.float32)
inputs[("inv_K",
scale)] = torch.from_numpy(inv_K).to(dtype=torch.float32)
## ZMH: load image with network-compatible size
for i in self.frame_idxs:
if i == "s":
other_side = {"r": "l", "l": "r"}[side]
inputs[("depth_gt_scale", i,
-1)], _, _ = self.get_depth(folder, frame_index,
other_side, do_flip)
else:
inputs[("depth_gt_scale", i,
-1)], velo_rect, P_rect_norm = self.get_depth(
folder, frame_index + i, side, do_flip)
if do_flip:
inputs[("velo_gt", i)] = flip_lidar(velo_rect, P_rect_norm)
inputs[("velo_gt", i)] = torch.from_numpy(
inputs[("velo_gt", i)].astype(np.float32))
else:
inputs[("velo_gt", i)] = torch.from_numpy(
velo_rect.astype(np.float32))
for j in range(self.num_scales):
new_w = self.width // (2**j)
new_h = self.height // (2**j)
depth_gt = project_lidar_to_img(velo_rect, P_rect_norm,
np.array([new_h, new_w]))
if do_flip:
depth_gt = np.fliplr(depth_gt)
### generate mask from the lidar points
# mask = binary_dilation(depth_gt, self.dilate_struct[j])
mask = depth_gt.copy()
mask[int(new_h / 2):] = 1
mask = binary_closing(mask, self.dilate_struct[j])
mask = np.expand_dims(mask, 0)
inputs[("depth_mask", i, j)] = torch.from_numpy(mask)
# depth_gt = skimage.transform.resize(inputs[("depth_gt_scale", i, -1)], (new_h, new_w), order=0, anti_aliasing=False ) # mine, ok
# depth_gt = skimage.transform.resize(inputs[("depth_gt_scale", i, -1)], (new_h, new_w), order=0, preserve_range=True, mode='constant') # from kitti_dataset.py (KITTIRAWDataset), not ok
# depth_gt = skimage.transform.resize(inputs[("depth_gt_scale", i, -1)], (new_h, new_w), order=0, preserve_range=True, mode='constant', anti_aliasing=False) # combined, ok
depth_gt = np.expand_dims(depth_gt, 0)
inputs[("depth_gt_scale", i,
j)] = torch.from_numpy(depth_gt.astype(np.float32))
inputs[("depth_mask_gt", i,
j)] = inputs[("depth_gt_scale", i, j)] > 0
depth_gt = np.expand_dims(inputs[("depth_gt_scale", i, -1)], 0)
inputs[("depth_gt_scale", i,
-1)] = torch.from_numpy(depth_gt.astype(np.float32))
def gt_depth_from_line(line, opt, data_path, mode="train"):
folder, frame_id, side = line.split()
frame_id = int(frame_id)
if mode == "train":
im_shape_predefined = [375, 1242]
if opt.eval_split == "eigen":
calib_dir = os.path.join(data_path, folder.split("/")[0])
velo_filename = os.path.join(data_path, folder, "velodyne_points/data",
"{:010d}.bin".format(frame_id))
if mode == "train":
velo_rect, P_rect_norm, im_shape = generate_depth_map(
calib_dir, velo_filename, 2)
gt_depth = project_lidar_to_img(
velo_rect, P_rect_norm, im_shape_predefined
) ## ZMH: the way gt is generated I used in training. Resize to a fixed size
gt_depth = np.expand_dims(gt_depth, 0)
gt_depth = np.expand_dims(gt_depth, 0)
gt_depth = torch.from_numpy(gt_depth.astype(np.float32))
else:
# gt_depth = generate_depth_map(calib_dir, velo_filename, 2, True) ## ZMH: This won't work because the generate_depth_map function has been redefined.
gt_depth = generate_depth_map_original(
calib_dir, velo_filename, 2, True
) ## ZMH: the original function in monodepth2, the size could be different for each img
# gt_depth = generate_depth_map_original(calib_dir, velo_filename, 2, False) ## ZMH: the original function in monodepth2, use transformed depth
elif opt.eval_split == "eigen_benchmark":
# gt_depth_path = os.path.join(opt.data_path, folder, "proj_depth",
# "groundtruth", "image_02", "{:010d}.png".format(frame_id))
gt_depth_path = os.path.join(data_path, folder, "proj_depth",
"groundtruth", "image_02",
"{:010d}.png".format(frame_id), 'val',
folder.split("/")[1], "proj_depth",
"groundtruth", "image_02",
"{:010d}.png".format(frame_id))
if not os.path.exists(gt_depth_path):
gt_depth_path = os.path.join(data_path, folder, "proj_depth",
"groundtruth", "image_02",
"{:010d}.png".format(frame_id),
'train',
folder.split("/")[1], "proj_depth",
"groundtruth", "image_02",
"{:010d}.png".format(frame_id))
if not os.path.exists(gt_depth_path):
raise ValueError("This file does not exist! {} {}".format(
folder, frame_id))
if mode != "train":
gt_depth = np.array(pil.open(gt_depth_path)).astype(
np.float32) / 256
else:
gt_depth = pil.open(gt_depth_path)
gt_depth = gt_depth.resize(im_shape_predefined[::-1], pil.NEAREST)
gt_depth = np.array(gt_depth).astype(np.float32) / 256
gt_depth = np.expand_dims(gt_depth, 0)
gt_depth = np.expand_dims(gt_depth, 0)
gt_depth = torch.from_numpy(gt_depth.astype(
np.float32)) # input original scale
elif opt.eval_split == "vkitti":
f_str = "depth_{:05d}.png".format(frame_id)
gt_depth_path = os.path.join(data_path, folder, "clone", "frames",
"depth", "Camera_0", f_str)
gt_depth = cv2.imread(gt_depth_path,
cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)
gt_depth = gt_depth.astype(np.float32) / 100
gt_depth[gt_depth > 80] = 0
height = gt_depth.shape[0]
gt_depth[:int(height / 2)] = 0
# print(gt_depth.max(), gt_depth.min())
if mode == "train":
gt_depth = np.expand_dims(gt_depth, 0)
gt_depth = np.expand_dims(gt_depth, 0)
gt_depth = torch.from_numpy(gt_depth.astype(np.float32))
else:
raise ValueError("opt.eval_split {} not recognized".format(
opt.eval_split))
return gt_depth
def __call__(self, line):
words = line.split()
folder = words[0]
frame_id = int(words[1])
side = words[2]
# print(line)
# for frame_id in range(frame_idx-1, frame_idx+2):
calib_path = os.path.join(self.data_path, folder.split("/")[0])
velo_filename = os.path.join(
self.data_path, folder,
"velodyne_points/data/{:010d}.bin".format(frame_id))
if not os.path.exists(velo_filename):
print("Not found:", velo_filename)
return
global counter
with counter.get_lock():
valid_1 = check_existence(
self.ext_data_path, folder, frame_id,
"unfilled_depth_0{}/data".format(self.side_map[side]))
valid_2 = check_existence(
self.ext_data_path, folder, frame_id,
"filled_depth_0{}/data".format(self.side_map[side]))
if valid_1 and valid_2:
counter.value += 1
i = counter.value
print(i, "skipped in {} s.".format(time.time() - start_time))
print(folder, frame_id, side)
return
velo_rect, P_rect_norm, im_shape = generate_depth_map(
calib_path, velo_filename, self.side_map[side])
depth_gt = project_lidar_to_img(
velo_rect, P_rect_norm,
self.full_res_shape[::-1]) # H*W, raw depth
# prep_path_and_filename(self.data_path, folder, frame_id, "unfilled_depth_0{}/data".format(self.side_map[side]), depth_gt)
prep_path_and_filename(
self.ext_data_path, folder, frame_id,
"unfilled_depth_0{}/data".format(self.side_map[side]), depth_gt)
### fill depth
image_path = os.path.join(
self.data_path, folder,
"image_0{}/data/{:010d}.jpg".format(self.side_map[side], frame_id))
with open(image_path, 'rb') as f:
with Image.open(f) as img:
rgb_img = img.convert('RGB')
rgb_img = rgb_img.resize(self.full_res_shape)
rgb_imgarray = np.array(rgb_img).astype(
np.float32) / 255 # H*W*3, between 0~1
# print(rgb_imgarray.min(), rgb_imgarray.max(), rgb_imgarray.shape, np.median(rgb_imgarray))
depth_filled = fill_depth_colorization(imgRgb=rgb_imgarray,
imgDepthInput=depth_gt,
alpha=1)
# prep_path_and_filename(self.data_path, folder, frame_id, "filled_depth_0{}/data".format(self.side_map[side]), depth_filled)
prep_path_and_filename(
self.ext_data_path, folder, frame_id,
"filled_depth_0{}/data".format(self.side_map[side]), depth_filled)
# += operation is not atomic, so we need to get a lock:
with counter.get_lock():
counter.value += 1
i = counter.value
if (i < 10) or (i % 10 == 0 and i < 100) or (
i % 100 == 0 and i < 1000) or (i % 1000 == 0
and i < 10000) or (i % 5000
== 0):
print(i, "finished in {} s.".format(time.time() - start_time))
print(folder, frame_id, side)