def try_image_to_kitti():
# loading data
directory = r'D:\output-datasets\offroad-14\1'
base_name = '000084'
rgb_file = os.path.join(directory, '{}.jpg'.format(base_name))
depth_file = os.path.join(directory, '{}-depth.tiff'.format(base_name))
stencil_file = os.path.join(directory, '{}-stencil.png'.format(base_name))
json_file = os.path.join(directory, '{}.json'.format(base_name))
rgb = np.array(Image.open(rgb_file))
depth = tifffile.imread(depth_file)
stencil = np.array(Image.open(stencil_file))
with open(json_file) as f:
data = json.load(f)
# creating pointcloud for original data
csv_name = base_name + '-orig'
vecs, _ = points_to_homo(data, depth, tresholding=False)
vecs_p = ndc_to_view(vecs, data['proj_matrix'])
vecs_p_world = view_to_world(vecs_p, np.array(data['view_matrix']))
a = np.asarray(vecs_p_world[0:3, :].T)
np.savetxt(os.path.join('kitti-format', "points-{}.csv".format(csv_name)), a, delimiter=",")
# whole gta to kitti transformation
rgb, depth, stencil = image_gta_to_kitti(rgb, depth, stencil, data['width'], data['height'], data['camera_fov'])
data['proj_matrix'] = get_kitti_proj_matrix(np.array(data['proj_matrix'])).tolist()
data['width'], data['height'] = get_kitti_img_size()
# saving new images
Image.fromarray(rgb).convert(mode="RGB").save(os.path.join('kitti-format', '{}.jpg'.format(base_name)))
tifffile.imsave(os.path.join('kitti-format', '{}-depth-orig.tiff'.format(base_name)), depth)
tifffile.imsave(os.path.join('kitti-format', '{}-depth-lzma.tiff'.format(base_name)), depth, compress='lzma')
tifffile.imsave(os.path.join('kitti-format', '{}-depth-zip-5.tiff'.format(base_name)), depth, compress=5)
tifffile.imsave(os.path.join('kitti-format', '{}-depth-zip-9.tiff'.format(base_name)), depth, compress=9)
tifffile.imsave(os.path.join('kitti-format', '{}-depth-zip-zstd.tiff'.format(base_name)), depth, compress='zstd')
Image.fromarray(stencil).save(os.path.join('kitti-format', '{}-stencil.jpg'.format(base_name)))
with open(os.path.join('kitti-format', '{}.json'.format(base_name)), 'w+') as f:
json.dump(data, f)
data['view_matrix'] = np.array(data['view_matrix'])
check_proj_matrices(depth, data)
# creating pointcloud for kitti format data
csv_name = base_name + '-kitti'
vecs, _ = points_to_homo(data, depth, tresholding=False)
vecs_p = ndc_to_view(vecs, data['proj_matrix'])
vecs_p_world = view_to_world(vecs_p, np.array(data['view_matrix']))
a = np.asarray(vecs_p_world[0:3, :].T)
np.savetxt(os.path.join('kitti-format', "points-{}.csv".format(csv_name)), a, delimiter=",")
def check_proj_matrices(depth, data):
calib = load_calibration_cam_to_cam('kitti-calibration-example/calib_cam_to_cam.txt')
kitti_proj_matrix = calib['P_rect'][2]
proj_matrix = np.array(data['proj_matrix'])
cropped_proj_matrix = proj_matrix[(0, 1, 3), :]
width, height = get_kitti_img_size()
to_pixel_matrix = np.array([
[width/2, 0, width/2],
[0, -height/2, height/2],
[0, 0, 1],
])
result = to_pixel_matrix @ cropped_proj_matrix
data['cam_far_clip'] = 1000
points_ndc, pixels = points_to_homo(data, depth, tresholding=True)
points_meters = ndc_to_view(points_ndc, proj_matrix)
points_meters[2, :] *= -1
points_pixels = np.zeros((3, points_ndc.shape[1]))
points_pixels[0] = pixels[1] * points_meters[2, :]
points_pixels[1] = pixels[0] * points_meters[2, :]
points_pixels[2] = points_meters[2, :]
# points_pixels must contain point values after the normalization (dividing by depth),
# so now they must be multiplied by depth
points_meters = points_meters[0:3, :]
calculated_kitti_proj, residuals, rank, s = np.linalg.lstsq(points_meters.T, points_pixels.T)
calculated_kitti_proj = calculated_kitti_proj.T # need to transpose it because of matrix multiplication from the other side
pass
def obtain_toyota_projection_matrix_from_image():
directory = r'D:\output-datasets\onroad-3'
base_name = '2018-07-31--17-45-30--020'
rgb_file = os.path.join(directory, '{}.jpg'.format(base_name))
depth_file = os.path.join(directory, '{}-depth.png'.format(base_name))
json_file = os.path.join(directory, '{}.json'.format(base_name))
depth = np.array(Image.open(depth_file))
depth = depth / np.iinfo(np.uint16).max # normalizing into NDC
with open(json_file, mode='r') as f:
data = json.load(f)
proj_matrix = np.array(data['proj_matrix'])
data['cam_far_clip'] = 1000
points_ndc, pixels = points_to_homo(data, depth, tresholding=True)
points_meters = ndc_to_view(points_ndc, proj_matrix)
points_meters *= 1000 # distances are calculated in mm
points_meters[2, :] *= -1
points_pixels = np.zeros((3, points_ndc.shape[1]))
points_pixels[0] = pixels[1] * points_meters[2, :]
points_pixels[1] = pixels[0] * points_meters[2, :]
points_pixels[2] = points_meters[2, :]
# points_pixels must contain point values after the normalization (dividing by depth),
# so now they must be multiplied by depth
points_meters = points_meters[0:3, :]
toyota_proj, residuals, rank, s = np.linalg.lstsq(points_meters.T,
points_pixels.T)
toyota_proj = toyota_proj.T # need to transpose it because of matrix multiplication from the other side
# centering to image check
assert np.isclose((data['width'] / 2) - 0.5, toyota_proj[0, 2])
assert np.isclose((data['height'] / 2) - 0.5, toyota_proj[1, 2])
# zeros where they should be checking
assert np.isclose(0, toyota_proj[0, 1], atol=2e-6)
assert np.isclose(0, toyota_proj[1, 0], atol=2e-6)
assert np.isclose(0, toyota_proj[2, 0], atol=2e-6)
assert np.isclose(0, toyota_proj[2, 1], atol=2e-6)
assert np.isclose(1, toyota_proj[2, 2])
# fov should be same for x and y
# fov in pixel size checking
assert np.isclose(toyota_proj[0, 0], -toyota_proj[1, 1], atol=5e-1)
print(toyota_proj.T, ' is the projection matrix')
print((toyota_proj[0, 0] - toyota_proj[1, 1]) / 2,
'is the fov you are looking for')
def camera_to_pointcloud(cam):
# print('MAX_DISTANCE', MAX_DISTANCE)
# start = time.time()
name = cam['imagepath']
depth = load_depth(name)
cam['cam_far_clip'] = MAX_DISTANCE
# print('load_depth:', time.time() - start)
# start = time.time()
vecs, _ = points_to_homo(cam, depth)
# print('points_to_homo:', time.time() - start)
# start = time.time()
assert(vecs.shape[0] == 4)
vecs_p = ndc_to_view(vecs, cam['proj_matrix'])
vecs_p_world = view_to_world(vecs_p, cam['view_matrix'])
# print('ndc_to_world:', time.time() - start)
assert(vecs_p_world.shape[0] == 4)
return vecs_p_world[0:3, :]
import os
from gta_math import points_to_homo, ndc_to_view, view_to_world
from PIL import Image
import json
def save_csv(vecs_p, name):
a = np.asarray(vecs_p[0:3, :].T)
np.savetxt("points-{}.csv".format(name), a, delimiter=",")
directory = r'D:\output-datasets\offroad-7\0'
file_name = '2018-08-13--11-15-01--499'
rgb_file = os.path.join(directory, '{}.jpg'.format(file_name))
depth_file = os.path.join(directory, '{}-depth.png'.format(file_name))
json_file = os.path.join(directory, '{}.json'.format(file_name))
rgb = np.array(Image.open(rgb_file))
depth = np.array(Image.open(depth_file))
depth = depth / np.iinfo(np.uint16).max # normalizing into NDC
with open(json_file, mode='r') as f:
data = json.load(f)
data['proj_matrix'] = np.array(data['proj_matrix'])
data['view_matrix'] = np.array(data['view_matrix'])
vecs, _ = points_to_homo(data, depth, tresholding=False)
vecs_p = ndc_to_view(vecs, np.array(data['proj_matrix']))
vecs_p_world = view_to_world(vecs_p, np.array(data['view_matrix']))
save_csv(vecs_p, 'my-points-'+file_name)