def create_seg_masks(variant_mask_files):
# Output paths and filenames
segmentation_dir_path = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'],
SUBFOLDER_MAP_SYNTHETIC['segmentation-masks']['folder-name'])
prefix = os.path.basename(variant_mask_files)[
0:0 - len(SUBFOLDER_MAP_SYNTHETIC['variant-masks']['postfix'])]
segmentation_mask_rectified_filename = (
prefix + SUBFOLDER_MAP_SYNTHETIC['segmentation-masks']['postfix'])
segmentation_mask_rectified_file = os.path.join(
segmentation_dir_path, segmentation_mask_rectified_filename)
# If outlines file already exists, skip
if Path(segmentation_mask_rectified_file).is_file():
return False
variant_mask = exr_loader(variant_mask_files, ndim=1)
seg_mask = np.zeros((variant_mask.shape), dtype=np.uint8)
seg_mask[variant_mask > 0] = 1
imageio.imwrite(segmentation_mask_rectified_file, seg_mask)
return True
def create_rectified_depth_image(path_rendered_depth_file, cos_matrix_y,
cos_matrix_x):
'''Creates and saves a rectified depth image from the rendered depth image
The rendered depth image contains depth of each pixel from the object to the camera center/lens. It is obtained
through techniques similar to ray tracing.
However, our algorithms (like creation of point clouds) expect the depth image to be in the same format as output
by stereo depth cameras. Stereo cameras output a depth image where the depth is calculated from the object to
camera plane (the plane is perpendicular to axis coming out of camera lens). Hence, if a flat wall is kept in front
of the camera perpendicular to it, the depth of each pixel on the wall contains the same depth value.
This is refered to as the rectified depth image.
/ -----
/
c--- c-----
\
\ -----
Rendered depth image Rectified Depth Image
Args:
depthpath_rendered_depth_file_file (str) : Path to the rendered depth image in .exr format with dtype=float32.
Each pixel contains depth from pixel to camera center/lens.
cos_matrix_y (numpy.ndarray): Shape (height, width) Matrix of cos of angle in the y-axis from image center
to each pixel in depth image.
cos_matrix_x (numpy.ndarray): Shape (height, width) Matrix of cos of angle in the x-axis from image center
to each pixel in depth image.
Returns:
bool: False if file exists and it skipped it. True if it created the depth rectified file
'''
# Output paths and filenames
outlines_dir_path = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'],
SUBFOLDER_MAP_SYNTHETIC['depth-files-rectified']['folder-name'])
prefix = os.path.basename(path_rendered_depth_file)[
0:0 - len(SUBFOLDER_MAP_SYNTHETIC['depth-files']['postfix'])]
output_depth_rectified_filename = (
prefix + SUBFOLDER_MAP_SYNTHETIC['depth-files-rectified']['postfix'])
output_depth_rectified_file = os.path.join(
outlines_dir_path, output_depth_rectified_filename)
# If outlines file already exists, skip
if Path(output_depth_rectified_file).is_file():
# print('file exists')
return False
depth_img = exr_loader(path_rendered_depth_file, ndim=1)
# calculate modified depth/pixel in mtrs
output = np.multiply(np.multiply(depth_img, cos_matrix_y), cos_matrix_x)
output = np.stack((output, output, output), axis=0)
exr_saver(output_depth_rectified_file, output, ndim=3)
return True
def calculate_cos_matrix(depth_img_path, fov_y=0.7428, fov_x=1.2112):
'''Calculates the cos of the angle between each pixel, camera center and center of image
First, it will take the angle in the x-axis from image center to each pixel, take the cos of each angle and store
in a matrix. Then it will do the same, except for angles in y-axis.
These cos matrices are used to recitify the depth image.
Args:
depth_img_path (str) : Path to the depth image in exr format
fov_y (float): FOV (Feild of View) of the camera along height of the image in radians, default=0.7428
fov_x (float): FOV (Feild of View) of the camera along width of the image in radians, default=1.2112
Returns:
cos_matrix_y (numpy.ndarray): Matrix of cos of angle in the y-axis from image center to each pixel in
the depth image.
Shape: (height, width)
cos_matrix_x (numpy.ndarray): Matrix of cos of angle in the x-axis from image center to each pixel in
the depth image.
Shape: (height, width)
'''
depth_img = exr_loader(depth_img_path, ndim=1)
height, width = depth_img.shape
center_y, center_x = (height / 2), (width / 2)
angle_per_pixel_along_y = fov_y / height # angle per pixel along height of the image
angle_per_pixel_along_x = fov_x / width # angle per pixel along width of the image
# create two static arrays to calculate focal angles along x and y axis
cos_matrix_y = np.zeros((height, width), 'float32')
cos_matrix_x = np.zeros((height, width), 'float32')
# calculate cos matrix along y - axis
for i in range(height):
for j in range(width):
angle = abs(center_y - (i)) * angle_per_pixel_along_y
cos_value = np.cos(angle)
cos_matrix_y[i][j] = cos_value
# calculate cos matrix along x-axis
for i in range(width):
for j in range(height):
angle = abs(center_x - (i)) * angle_per_pixel_along_x
cos_value = np.cos(angle)
cos_matrix_x[j][i] = cos_value
return cos_matrix_y, cos_matrix_x
sobelxy_binary3d = np.array(sobelxy_list).transpose((1, 2, 0))
sobelxy_binary3d = sobelxy_binary3d.astype(np.uint8) * 255
sobelxy_binary = np.zeros(
(surface_normal_rgb16.shape[1], surface_normal_rgb16.shape[2]))
for channel in sobelxy_list:
sobelxy_binary[channel > 0] = 255
# print('normal nonzero:', np.sum((edges_sobel_binary > 0) & (edges_sobel_binary < 255)))
return sobelxy_binary
for i in range(100, 2751):
# Load Depth Img convert to outlines and resize
print('Loading img %d' % (i))
depth_img_orig = exr_loader(depth_path % (i), ndim=1)
depth_edges = outline_from_depth(depth_img_orig)
depth_edges_img = Image.fromarray(depth_edges, 'L').resize(
(width, height), resample=Image.NEAREST)
depth_edges = np.asarray(depth_edges_img)
# Load RGB image, convert to outlines and resize
surface_normal = exr_loader(normals_path % (i))
normals_edges = outline_from_normal(surface_normal)
# edges = Image.fromarray(edges).resize((224,224))
save_output = True
if (save_output):
depth_edges_img.save(path_save_depth_edges % (i))
imsave(path_save_normal_edges % (i), normals_edges)
def preprocess_normals(normals_path, imsize):
"""Resize Normals and save as exr file
Args:
normals_path (str) = The path to an exr file that contains the normal to be preprocessed.
imsize (tuple, int) = (height, width) The size to which image is to be resized.
Returns:
bool: False if file exists and it skipped it. True if it converted the file.
"""
if len(imsize) != 2:
raise ValueError(
'Pass imsize as a tuple of (height, width). Given imsize = {}'.
format(imsize))
prefix = os.path.basename(normals_path)[
0:0 - len(SUBFOLDER_MAP_SYNTHETIC['camera-normals']['postfix'])]
preprocess_normals_dir = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['training-data'],
SUBFOLDER_MAP_RESIZED_SYNTHETIC['preprocessed-camera-normals']
['folder-name'])
preprocess_normal_viz_dir = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['training-data'],
SUBFOLDER_MAP_RESIZED_SYNTHETIC['preprocessed-camera-normals-rgb']
['folder-name'])
preprocess_normals_filename = prefix + SUBFOLDER_MAP_RESIZED_SYNTHETIC[
'preprocessed-camera-normals']['postfix']
preprocess_normal_viz_filename = prefix + (SUBFOLDER_MAP_RESIZED_SYNTHETIC[
'preprocessed-camera-normals-rgb']['postfix'])
output_file = os.path.join(preprocess_normals_dir,
preprocess_normals_filename)
output_rgb_file = os.path.join(preprocess_normal_viz_dir,
preprocess_normal_viz_filename)
if Path(output_file).is_file() and Path(
output_rgb_file).is_file(): # file exists
return False
normals = exr_loader(normals_path, ndim=3)
# Resize the normals
normals = normals.transpose(1, 2, 0)
normals_resized = resize(normals,
imsize,
anti_aliasing=True,
clip=True,
mode='reflect')
normals_resized = normals_resized.transpose(2, 0, 1)
# Normalize the normals
normals = torch.from_numpy(normals_resized)
normals = nn.functional.normalize(normals, p=2, dim=0)
normals = normals.numpy()
# Save array as EXR file
exr_saver(output_file, normals, ndim=3)
# Output converted Normals as RGB images
camera_normal_rgb = normal_to_rgb(normals.transpose(1, 2, 0))
imageio.imwrite(output_rgb_file, camera_normal_rgb)
return True
def create_outlines_training_data(path_depth_file, path_camera_normal_file):
'''Creates training data for the Outlines Prediction Model
It creates outlines from the depth image and surface normal image.
Places where Depth and Normal outlines overlap, priority is given to depth pixels.
Expects the depth image to be in .exr format, with dtype=float32 where each pixel represents the depth in meters
Expects the surfacte normal image to be in .exr format, with dtype=float32. Each pixel contains the
surface normal, RGB channels mapped to XYZ axes.
Args:
path_depth_file (str): Path to the depth image.
path_camera_normal_file (str): Path to the surface normals image.
Returns:
bool: False if file exists and it skipped it. True if it created the outlines file
'''
# Output paths and filenames
outlines_dir_path = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'],
SUBFOLDER_MAP_SYNTHETIC['outlines']['folder-name'])
outlines_rgb_dir_path = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'],
SUBFOLDER_MAP_SYNTHETIC['outlines-rgb']['folder-name'])
prefix = os.path.basename(path_depth_file)[
0:0 - len(SUBFOLDER_MAP_SYNTHETIC['depth-files-rectified']['postfix'])]
output_outlines_filename = (prefix +
SUBFOLDER_MAP_SYNTHETIC['outlines']['postfix'])
outlines_rgb_filename = (
prefix + SUBFOLDER_MAP_SYNTHETIC['outlines-rgb']['postfix'])
output_outlines_file = os.path.join(outlines_dir_path,
output_outlines_filename)
output_outlines_rgb_file = os.path.join(outlines_rgb_dir_path,
outlines_rgb_filename)
# If outlines file already exists, skip
if Path(output_outlines_file).is_file() and Path(
output_outlines_rgb_file).is_file():
return False
# Create outlines from depth image
depth_img_orig = exr_loader(path_depth_file, ndim=1)
depth_edges = outlines_from_depth(depth_img_orig)
# Create outlines from surface normals
surface_normal = exr_loader(path_camera_normal_file)
normals_edges = outline_from_normal(surface_normal)
# Depth and Normal outlines should not overlap. Priority given to depth.
normals_edges[depth_edges == 255] = 0
# Modified edges and create mask
assert (depth_edges.shape == normals_edges.shape
), " depth and cameral normal shapes are different"
height, width = depth_edges.shape
output = np.zeros((height, width), 'uint8')
output[normals_edges == 255] = 2
output[depth_edges == 255] = 1
# Removes extraneous outlines near the border of the image
# In our outlines image, the borders of the image contain depth and/or surface normal outlines, where there are none
# The cause is unknown, we remove them by setting all pixels near border to background class.
num_of_rows_to_delete_y_axis = 6
output[:num_of_rows_to_delete_y_axis, :] = 0
output[-num_of_rows_to_delete_y_axis:, :] = 0
num_of_rows_to_delete_x_axis = 6
output[:, :num_of_rows_to_delete_x_axis] = 0
output[:, -num_of_rows_to_delete_x_axis:] = 0
# Save the outlines
imageio.imwrite(output_outlines_file, output)
output_color = label_to_rgb(output)
imageio.imwrite(output_outlines_rgb_file, output_color)
return True
def preprocess_world_to_camera_normals(path_world_normals_file,
path_json_file):
'''Will convert normals from World co-ords to Camera co-ords
It will create a folder to store converted files. A quaternion for conversion of normal from world to camera
co-ords is read from the json file and is multiplied with each normal in source file.
Args:
path_world_normals_file (str): Path to world co-ord normals file.
path_json_file (str): Path to json file which stores quaternion.
Returns:
bool: False if file exists and it skipped it. True if it converted the file.
'''
# Output paths and filenames
camera_normal_dir_path = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'],
SUBFOLDER_MAP_SYNTHETIC['camera-normals']['folder-name'])
camera_normal_rgb_dir_path = os.path.join(
NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'],
SUBFOLDER_MAP_SYNTHETIC['camera-normals-rgb']['folder-name'])
prefix = os.path.basename(path_world_normals_file)[
0:0 - len(SUBFOLDER_MAP_SYNTHETIC['world-normals']['postfix'])]
output_camera_normal_filename = (
prefix + SUBFOLDER_MAP_SYNTHETIC['camera-normals']['postfix'])
camera_normal_rgb_filename = (
prefix + SUBFOLDER_MAP_SYNTHETIC['camera-normals-rgb']['postfix'])
output_camera_normal_file = os.path.join(camera_normal_dir_path,
output_camera_normal_filename)
camera_normal_rgb_file = os.path.join(camera_normal_rgb_dir_path,
camera_normal_rgb_filename)
# If cam normal already exists, skip
if Path(output_camera_normal_file).is_file():
return False
world_normal_file = os.path.join(
SUBFOLDER_MAP_SYNTHETIC['world-normals']['folder-name'],
os.path.basename(path_world_normals_file))
camera_normal_file = os.path.join(
SUBFOLDER_MAP_SYNTHETIC['camera-normals']['folder-name'],
prefix + SUBFOLDER_MAP_SYNTHETIC['camera-normals']['postfix'])
# print(" Converting {} to {}".format(world_normal_file, camera_normal_file))
# Read EXR File
exr_np = exr_loader(path_world_normals_file)
exr_x, exr_y, exr_z = exr_np[0], exr_np[1], exr_np[2]
assert (exr_x.shape == exr_y.shape)
assert (exr_y.shape == exr_z.shape)
# Read Camera's Inverse Quaternion
json_file = open(path_json_file)
data = json.load(json_file)
inverted_camera_quaternation = np.asarray(
data['camera']['world_pose']['rotation']['inverted_quaternion'],
dtype=np.float32)
# Convert Normals to Camera Space
camera_normal = world_to_camera_normals(inverted_camera_quaternation,
exr_x, exr_y, exr_z)
# Output Converted Surface Normal Files
exr_arr = camera_normal.transpose((2, 0, 1))
exr_saver(output_camera_normal_file, exr_arr, ndim=3)
# Output Converted Normals as RGB images for visualization
camera_normal_rgb = normal_to_rgb(camera_normal)
imageio.imwrite(camera_normal_rgb_file, camera_normal_rgb)
return True
def main():
'''Converts dataset of float32 depth.exr images to scaled 16-bit png images with holes
This script takes in a dataset of depth images in a float32 .exr format.
Then it cuts out a hole in each and converts to a scaled uint16 png image.
These modified depth images are used as input to the depth2depth module.
'''
parser = argparse.ArgumentParser(description='Dataset Directory path')
parser.add_argument('-p',
'--depth-path',
required=True,
help='Path to directory containing depth images',
metavar='path/to/dataset')
parser.add_argument('-l',
'--height',
help='The height of output image',
type=int,
default=288)
parser.add_argument('-w',
'--width',
help='The width of output image',
type=int,
default=512)
args = parser.parse_args()
# create a directory for depth scaled png images, if it doesn't exist
depth_imgs = os.path.join(args.depth_path, 'input-depth-scaled')
if not os.path.isdir(depth_imgs):
os.makedirs(depth_imgs)
print(" Created dir:", depth_imgs)
else:
print(" Output Dir Already Exists:", depth_imgs)
print(" Will overwrite files within")
# read the exr file as np array, scale it and store as png image
scale_value = 4000
print(
'Converting depth files from exr format to a scaled uin16 png format...'
)
print(
'Will make a portion of the img zero during conversion to test depth2depth executable'
)
for root, dirs, files in os.walk(args.depth_path):
for filename in sorted(fnmatch.filter(files, '*depth.exr')):
name = filename[:-4] + '.png'
np_image = exr_loader(os.path.join(args.depth_path, filename),
ndim=1)
height, width = np_image.shape
# Create a small rectangular hole in input depth, to be filled in by depth2depth module
h_start, h_stop = (height // 8) * 2, (height // 8) * 6
w_start, w_stop = (width // 8) * 5, (width // 8) * 7
# Make half the image zero for testing depth2depth
np_image[h_start:h_stop, w_start:w_stop] = 0.0
# Scale the depth to create the png file for depth2depth
np_image = np_image * scale_value
np_image = np_image.astype(np.uint16)
# Convert to PIL
array_buffer = np_image.tobytes()
img = Image.new("I", np_image.T.shape)
img.frombytes(array_buffer, 'raw', 'I;16')
# Resize and save
img = img.resize((args.width, args.height), Image.BILINEAR)
img.save(os.path.join(depth_imgs, name))
print(
'total ',
len([
name for name in os.listdir(depth_imgs)
if os.path.isfile(os.path.join(depth_imgs, name))
]), ' converted from exr to png')