def create_seg_masks(variant_mask_files): # Output paths and filenames segmentation_dir_path = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'], SUBFOLDER_MAP_SYNTHETIC['segmentation-masks']['folder-name']) prefix = os.path.basename(variant_mask_files)[ 0:0 - len(SUBFOLDER_MAP_SYNTHETIC['variant-masks']['postfix'])] segmentation_mask_rectified_filename = ( prefix + SUBFOLDER_MAP_SYNTHETIC['segmentation-masks']['postfix']) segmentation_mask_rectified_file = os.path.join( segmentation_dir_path, segmentation_mask_rectified_filename) # If outlines file already exists, skip if Path(segmentation_mask_rectified_file).is_file(): return False variant_mask = exr_loader(variant_mask_files, ndim=1) seg_mask = np.zeros((variant_mask.shape), dtype=np.uint8) seg_mask[variant_mask > 0] = 1 imageio.imwrite(segmentation_mask_rectified_file, seg_mask) return True
def create_rectified_depth_image(path_rendered_depth_file, cos_matrix_y, cos_matrix_x): '''Creates and saves a rectified depth image from the rendered depth image The rendered depth image contains depth of each pixel from the object to the camera center/lens. It is obtained through techniques similar to ray tracing. However, our algorithms (like creation of point clouds) expect the depth image to be in the same format as output by stereo depth cameras. Stereo cameras output a depth image where the depth is calculated from the object to camera plane (the plane is perpendicular to axis coming out of camera lens). Hence, if a flat wall is kept in front of the camera perpendicular to it, the depth of each pixel on the wall contains the same depth value. This is refered to as the rectified depth image. / ----- / c--- c----- \ \ ----- Rendered depth image Rectified Depth Image Args: depthpath_rendered_depth_file_file (str) : Path to the rendered depth image in .exr format with dtype=float32. Each pixel contains depth from pixel to camera center/lens. cos_matrix_y (numpy.ndarray): Shape (height, width) Matrix of cos of angle in the y-axis from image center to each pixel in depth image. cos_matrix_x (numpy.ndarray): Shape (height, width) Matrix of cos of angle in the x-axis from image center to each pixel in depth image. Returns: bool: False if file exists and it skipped it. True if it created the depth rectified file ''' # Output paths and filenames outlines_dir_path = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'], SUBFOLDER_MAP_SYNTHETIC['depth-files-rectified']['folder-name']) prefix = os.path.basename(path_rendered_depth_file)[ 0:0 - len(SUBFOLDER_MAP_SYNTHETIC['depth-files']['postfix'])] output_depth_rectified_filename = ( prefix + SUBFOLDER_MAP_SYNTHETIC['depth-files-rectified']['postfix']) output_depth_rectified_file = os.path.join( outlines_dir_path, output_depth_rectified_filename) # If outlines file already exists, skip if Path(output_depth_rectified_file).is_file(): # print('file exists') return False depth_img = exr_loader(path_rendered_depth_file, ndim=1) # calculate modified depth/pixel in mtrs output = np.multiply(np.multiply(depth_img, cos_matrix_y), cos_matrix_x) output = np.stack((output, output, output), axis=0) exr_saver(output_depth_rectified_file, output, ndim=3) return True
def calculate_cos_matrix(depth_img_path, fov_y=0.7428, fov_x=1.2112): '''Calculates the cos of the angle between each pixel, camera center and center of image First, it will take the angle in the x-axis from image center to each pixel, take the cos of each angle and store in a matrix. Then it will do the same, except for angles in y-axis. These cos matrices are used to recitify the depth image. Args: depth_img_path (str) : Path to the depth image in exr format fov_y (float): FOV (Feild of View) of the camera along height of the image in radians, default=0.7428 fov_x (float): FOV (Feild of View) of the camera along width of the image in radians, default=1.2112 Returns: cos_matrix_y (numpy.ndarray): Matrix of cos of angle in the y-axis from image center to each pixel in the depth image. Shape: (height, width) cos_matrix_x (numpy.ndarray): Matrix of cos of angle in the x-axis from image center to each pixel in the depth image. Shape: (height, width) ''' depth_img = exr_loader(depth_img_path, ndim=1) height, width = depth_img.shape center_y, center_x = (height / 2), (width / 2) angle_per_pixel_along_y = fov_y / height # angle per pixel along height of the image angle_per_pixel_along_x = fov_x / width # angle per pixel along width of the image # create two static arrays to calculate focal angles along x and y axis cos_matrix_y = np.zeros((height, width), 'float32') cos_matrix_x = np.zeros((height, width), 'float32') # calculate cos matrix along y - axis for i in range(height): for j in range(width): angle = abs(center_y - (i)) * angle_per_pixel_along_y cos_value = np.cos(angle) cos_matrix_y[i][j] = cos_value # calculate cos matrix along x-axis for i in range(width): for j in range(height): angle = abs(center_x - (i)) * angle_per_pixel_along_x cos_value = np.cos(angle) cos_matrix_x[j][i] = cos_value return cos_matrix_y, cos_matrix_x
sobelxy_binary3d = np.array(sobelxy_list).transpose((1, 2, 0)) sobelxy_binary3d = sobelxy_binary3d.astype(np.uint8) * 255 sobelxy_binary = np.zeros( (surface_normal_rgb16.shape[1], surface_normal_rgb16.shape[2])) for channel in sobelxy_list: sobelxy_binary[channel > 0] = 255 # print('normal nonzero:', np.sum((edges_sobel_binary > 0) & (edges_sobel_binary < 255))) return sobelxy_binary for i in range(100, 2751): # Load Depth Img convert to outlines and resize print('Loading img %d' % (i)) depth_img_orig = exr_loader(depth_path % (i), ndim=1) depth_edges = outline_from_depth(depth_img_orig) depth_edges_img = Image.fromarray(depth_edges, 'L').resize( (width, height), resample=Image.NEAREST) depth_edges = np.asarray(depth_edges_img) # Load RGB image, convert to outlines and resize surface_normal = exr_loader(normals_path % (i)) normals_edges = outline_from_normal(surface_normal) # edges = Image.fromarray(edges).resize((224,224)) save_output = True if (save_output): depth_edges_img.save(path_save_depth_edges % (i)) imsave(path_save_normal_edges % (i), normals_edges)
def preprocess_normals(normals_path, imsize): """Resize Normals and save as exr file Args: normals_path (str) = The path to an exr file that contains the normal to be preprocessed. imsize (tuple, int) = (height, width) The size to which image is to be resized. Returns: bool: False if file exists and it skipped it. True if it converted the file. """ if len(imsize) != 2: raise ValueError( 'Pass imsize as a tuple of (height, width). Given imsize = {}'. format(imsize)) prefix = os.path.basename(normals_path)[ 0:0 - len(SUBFOLDER_MAP_SYNTHETIC['camera-normals']['postfix'])] preprocess_normals_dir = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['training-data'], SUBFOLDER_MAP_RESIZED_SYNTHETIC['preprocessed-camera-normals'] ['folder-name']) preprocess_normal_viz_dir = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['training-data'], SUBFOLDER_MAP_RESIZED_SYNTHETIC['preprocessed-camera-normals-rgb'] ['folder-name']) preprocess_normals_filename = prefix + SUBFOLDER_MAP_RESIZED_SYNTHETIC[ 'preprocessed-camera-normals']['postfix'] preprocess_normal_viz_filename = prefix + (SUBFOLDER_MAP_RESIZED_SYNTHETIC[ 'preprocessed-camera-normals-rgb']['postfix']) output_file = os.path.join(preprocess_normals_dir, preprocess_normals_filename) output_rgb_file = os.path.join(preprocess_normal_viz_dir, preprocess_normal_viz_filename) if Path(output_file).is_file() and Path( output_rgb_file).is_file(): # file exists return False normals = exr_loader(normals_path, ndim=3) # Resize the normals normals = normals.transpose(1, 2, 0) normals_resized = resize(normals, imsize, anti_aliasing=True, clip=True, mode='reflect') normals_resized = normals_resized.transpose(2, 0, 1) # Normalize the normals normals = torch.from_numpy(normals_resized) normals = nn.functional.normalize(normals, p=2, dim=0) normals = normals.numpy() # Save array as EXR file exr_saver(output_file, normals, ndim=3) # Output converted Normals as RGB images camera_normal_rgb = normal_to_rgb(normals.transpose(1, 2, 0)) imageio.imwrite(output_rgb_file, camera_normal_rgb) return True
def create_outlines_training_data(path_depth_file, path_camera_normal_file): '''Creates training data for the Outlines Prediction Model It creates outlines from the depth image and surface normal image. Places where Depth and Normal outlines overlap, priority is given to depth pixels. Expects the depth image to be in .exr format, with dtype=float32 where each pixel represents the depth in meters Expects the surfacte normal image to be in .exr format, with dtype=float32. Each pixel contains the surface normal, RGB channels mapped to XYZ axes. Args: path_depth_file (str): Path to the depth image. path_camera_normal_file (str): Path to the surface normals image. Returns: bool: False if file exists and it skipped it. True if it created the outlines file ''' # Output paths and filenames outlines_dir_path = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'], SUBFOLDER_MAP_SYNTHETIC['outlines']['folder-name']) outlines_rgb_dir_path = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'], SUBFOLDER_MAP_SYNTHETIC['outlines-rgb']['folder-name']) prefix = os.path.basename(path_depth_file)[ 0:0 - len(SUBFOLDER_MAP_SYNTHETIC['depth-files-rectified']['postfix'])] output_outlines_filename = (prefix + SUBFOLDER_MAP_SYNTHETIC['outlines']['postfix']) outlines_rgb_filename = ( prefix + SUBFOLDER_MAP_SYNTHETIC['outlines-rgb']['postfix']) output_outlines_file = os.path.join(outlines_dir_path, output_outlines_filename) output_outlines_rgb_file = os.path.join(outlines_rgb_dir_path, outlines_rgb_filename) # If outlines file already exists, skip if Path(output_outlines_file).is_file() and Path( output_outlines_rgb_file).is_file(): return False # Create outlines from depth image depth_img_orig = exr_loader(path_depth_file, ndim=1) depth_edges = outlines_from_depth(depth_img_orig) # Create outlines from surface normals surface_normal = exr_loader(path_camera_normal_file) normals_edges = outline_from_normal(surface_normal) # Depth and Normal outlines should not overlap. Priority given to depth. normals_edges[depth_edges == 255] = 0 # Modified edges and create mask assert (depth_edges.shape == normals_edges.shape ), " depth and cameral normal shapes are different" height, width = depth_edges.shape output = np.zeros((height, width), 'uint8') output[normals_edges == 255] = 2 output[depth_edges == 255] = 1 # Removes extraneous outlines near the border of the image # In our outlines image, the borders of the image contain depth and/or surface normal outlines, where there are none # The cause is unknown, we remove them by setting all pixels near border to background class. num_of_rows_to_delete_y_axis = 6 output[:num_of_rows_to_delete_y_axis, :] = 0 output[-num_of_rows_to_delete_y_axis:, :] = 0 num_of_rows_to_delete_x_axis = 6 output[:, :num_of_rows_to_delete_x_axis] = 0 output[:, -num_of_rows_to_delete_x_axis:] = 0 # Save the outlines imageio.imwrite(output_outlines_file, output) output_color = label_to_rgb(output) imageio.imwrite(output_outlines_rgb_file, output_color) return True
def preprocess_world_to_camera_normals(path_world_normals_file, path_json_file): '''Will convert normals from World co-ords to Camera co-ords It will create a folder to store converted files. A quaternion for conversion of normal from world to camera co-ords is read from the json file and is multiplied with each normal in source file. Args: path_world_normals_file (str): Path to world co-ord normals file. path_json_file (str): Path to json file which stores quaternion. Returns: bool: False if file exists and it skipped it. True if it converted the file. ''' # Output paths and filenames camera_normal_dir_path = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'], SUBFOLDER_MAP_SYNTHETIC['camera-normals']['folder-name']) camera_normal_rgb_dir_path = os.path.join( NEW_DATASET_PATHS['root'], NEW_DATASET_PATHS['source-files'], SUBFOLDER_MAP_SYNTHETIC['camera-normals-rgb']['folder-name']) prefix = os.path.basename(path_world_normals_file)[ 0:0 - len(SUBFOLDER_MAP_SYNTHETIC['world-normals']['postfix'])] output_camera_normal_filename = ( prefix + SUBFOLDER_MAP_SYNTHETIC['camera-normals']['postfix']) camera_normal_rgb_filename = ( prefix + SUBFOLDER_MAP_SYNTHETIC['camera-normals-rgb']['postfix']) output_camera_normal_file = os.path.join(camera_normal_dir_path, output_camera_normal_filename) camera_normal_rgb_file = os.path.join(camera_normal_rgb_dir_path, camera_normal_rgb_filename) # If cam normal already exists, skip if Path(output_camera_normal_file).is_file(): return False world_normal_file = os.path.join( SUBFOLDER_MAP_SYNTHETIC['world-normals']['folder-name'], os.path.basename(path_world_normals_file)) camera_normal_file = os.path.join( SUBFOLDER_MAP_SYNTHETIC['camera-normals']['folder-name'], prefix + SUBFOLDER_MAP_SYNTHETIC['camera-normals']['postfix']) # print(" Converting {} to {}".format(world_normal_file, camera_normal_file)) # Read EXR File exr_np = exr_loader(path_world_normals_file) exr_x, exr_y, exr_z = exr_np[0], exr_np[1], exr_np[2] assert (exr_x.shape == exr_y.shape) assert (exr_y.shape == exr_z.shape) # Read Camera's Inverse Quaternion json_file = open(path_json_file) data = json.load(json_file) inverted_camera_quaternation = np.asarray( data['camera']['world_pose']['rotation']['inverted_quaternion'], dtype=np.float32) # Convert Normals to Camera Space camera_normal = world_to_camera_normals(inverted_camera_quaternation, exr_x, exr_y, exr_z) # Output Converted Surface Normal Files exr_arr = camera_normal.transpose((2, 0, 1)) exr_saver(output_camera_normal_file, exr_arr, ndim=3) # Output Converted Normals as RGB images for visualization camera_normal_rgb = normal_to_rgb(camera_normal) imageio.imwrite(camera_normal_rgb_file, camera_normal_rgb) return True
def main(): '''Converts dataset of float32 depth.exr images to scaled 16-bit png images with holes This script takes in a dataset of depth images in a float32 .exr format. Then it cuts out a hole in each and converts to a scaled uint16 png image. These modified depth images are used as input to the depth2depth module. ''' parser = argparse.ArgumentParser(description='Dataset Directory path') parser.add_argument('-p', '--depth-path', required=True, help='Path to directory containing depth images', metavar='path/to/dataset') parser.add_argument('-l', '--height', help='The height of output image', type=int, default=288) parser.add_argument('-w', '--width', help='The width of output image', type=int, default=512) args = parser.parse_args() # create a directory for depth scaled png images, if it doesn't exist depth_imgs = os.path.join(args.depth_path, 'input-depth-scaled') if not os.path.isdir(depth_imgs): os.makedirs(depth_imgs) print(" Created dir:", depth_imgs) else: print(" Output Dir Already Exists:", depth_imgs) print(" Will overwrite files within") # read the exr file as np array, scale it and store as png image scale_value = 4000 print( 'Converting depth files from exr format to a scaled uin16 png format...' ) print( 'Will make a portion of the img zero during conversion to test depth2depth executable' ) for root, dirs, files in os.walk(args.depth_path): for filename in sorted(fnmatch.filter(files, '*depth.exr')): name = filename[:-4] + '.png' np_image = exr_loader(os.path.join(args.depth_path, filename), ndim=1) height, width = np_image.shape # Create a small rectangular hole in input depth, to be filled in by depth2depth module h_start, h_stop = (height // 8) * 2, (height // 8) * 6 w_start, w_stop = (width // 8) * 5, (width // 8) * 7 # Make half the image zero for testing depth2depth np_image[h_start:h_stop, w_start:w_stop] = 0.0 # Scale the depth to create the png file for depth2depth np_image = np_image * scale_value np_image = np_image.astype(np.uint16) # Convert to PIL array_buffer = np_image.tobytes() img = Image.new("I", np_image.T.shape) img.frombytes(array_buffer, 'raw', 'I;16') # Resize and save img = img.resize((args.width, args.height), Image.BILINEAR) img.save(os.path.join(depth_imgs, name)) print( 'total ', len([ name for name in os.listdir(depth_imgs) if os.path.isfile(os.path.join(depth_imgs, name)) ]), ' converted from exr to png')