def read_dir(dir_path, scale=False): disp0_path = path.join(dir_path, 'disp0.pfm') disp1_path = path.join(dir_path, 'disp1.pfm') im0_path = path.join(dir_path, 'im0.png') im1_path = path.join(dir_path, 'im1.png') im1E_path = path.join(dir_path, 'im1E.png') im1L_path = path.join(dir_path, 'im1L.png') calib_path = path.join(dir_path, 'calib.txt') print('Loading disparities') disp0 = utils.readPFM(disp0_path)[0] disp1 = utils.readPFM(disp1_path)[0] print('Loading images') im0 = np.array(Image.open(im0_path).convert('RGB')) im1 = np.array(Image.open(im1_path).convert('RGB')) im1E = np.array(Image.open(im1E_path).convert('RGB')) im1L = np.array(Image.open(im1L_path).convert('RGB')) if scale: print('Scaling Images') im0 = imresize(im0, IMSIZE + im0.shape[2:]) im1 = imresize(im1, IMSIZE + im1.shape[2:]) im1E = imresize(im1E, IMSIZE + im1E.shape[2:]) im1L = imresize(im1L, IMSIZE + im1L.shape[2:]) print('Loading calib') calib = utils.read_calib(calib_path) b = calib['baseline'] f = calib['cam0'][0, 0] d = calib['doffs'] print('Creating depth maps') d0 = b * f / (disp0 + d) d1 = b * f / (disp1 + d) d0 = preprocess(d0) d1 = preprocess(d1) if scale: print('Scaling depth maps') d0 = imresize(d0, IMSIZE)[..., None] d1 = imresize(d1, IMSIZE)[..., None] return im0, im1, im1E, im1L, d0, d1
def load_batch(self, left, right, labels, is_training): batch_left = [] batch_right = [] batch_label = [] for x, y, z in zip(left, right, labels): # print(x) if is_training: crop_x = random.randint( 0, self.img_height - 1 - self.patch_size[0]) crop_y = random.randint( 0, self.img_width - 1 - self.patch_size[1]) else: crop_x = (self.img_height - 1 - self.patch_size[0]) // 2 crop_y = (self.img_width - 1 - self.patch_size[1]) // 2 x = cv2.imread(x) assert x.shape[:2] == (self.img_height, self.img_width) x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB) x = x[crop_x:crop_x + self.patch_size[0], crop_y:crop_y + self.patch_size[1], :] x = utils.mean_std(x) batch_left.append(x) y = cv2.imread(y) y = cv2.cvtColor(y, cv2.COLOR_BGR2RGB) y = y[crop_x:crop_x + self.patch_size[0], crop_y:crop_y + self.patch_size[1], :] y = utils.mean_std(y) batch_right.append(y) z = readPFM(z) # z = cv2.cvtColor(z, cv2.COLOR_BGR2GRAY) z = z[crop_x:crop_x + self.patch_size[0], crop_y:crop_y + self.patch_size[1]] z[z > (self.max_disp - 1)] = self.max_disp - 1 batch_label.append(z) return batch_left, batch_right, batch_label
def gen_scene_stereo_refocus(input_seg_obj_list, input_stereo_left_list, input_stereo_right_list, input_stereo_left_disp_list, input_stereo_right_disp_list, validity_test_function=None): """ Generates a random scene and render lef right stereo views, disparity map, objects masks, and refocused image. No check on the final scene is performed. """ assert (len(input_stereo_left_list) == len(input_stereo_right_list) == len(input_stereo_left_disp_list) == len(input_stereo_right_disp_list)) #open background image and disparity background_idx = random.randint(0, len(input_stereo_left_list) - 1) background_image_left = np.asarray( Image.open(input_stereo_left_list[background_idx])) background_image_right = np.asarray( Image.open(input_stereo_right_list[background_idx])) target_h = background_image_left.shape[0] target_w = background_image_right.shape[1] if background_image_left.shape[2] == 3: background_image_left = np.concatenate( [background_image_left, 255 * np.ones([target_h, target_w, 1])], -1) if background_image_right.shape[2] == 3: background_image_right = np.concatenate( [background_image_right, 255 * np.ones([target_h, target_w, 1])], -1) background_disparity_left = readPFM( input_stereo_left_disp_list[background_idx]) background_disparity_right = readPFM( input_stereo_right_disp_list[background_idx]) #generate randomness for objects nb_objects = random.randint(min_nb_objects, max_nb_objects) objects_scales = [ random.uniform(min_scale_objects, max_scale_objects) for x in range(nb_objects) ] objects_idxs = [ random.randint(0, len(input_seg_obj_list) - 1) for x in range(nb_objects) ] objects_disparities = [ random.uniform(min_disp_objects, max_disp_objects) for x in range(nb_objects) ] objects_dx = [ round(target_w * random.uniform(min_x_objects, max_x_objects)) for x in range(nb_objects) ] objects_dy = [ round(target_h * random.uniform(min_y_objects, max_y_objects)) for x in range(nb_objects) ] #FIXME: do integer translation? objects_rot = [ random.randint(min_object_rotation, max_object_rotation) for x in range(nb_objects) ] #open and transform the images object_images_left = [] object_disparity_map_left = [] object_disparity_map_noaplha_left = [] object_images_right = [] object_disparity_map_right = [] for i in range(nb_objects): object_image = Image.open(input_seg_obj_list[objects_idxs[i]]) #object_image.save("out/ORIG_%d.png"%i) #Image.fromarray(np.asarray(object_image)[:,:,3]).save("alpha_%d.png"%i) #resize image object_image = object_image.resize( (int(object_image.size[0] * objects_scales[i]), int(object_image.size[1] * objects_scales[i]))) #object_image.save("out/RESIZE_%d.png"%i) #pad to canvas image size so we dont have to worry about corpping accidentatluy things object_image = pad_to_size(object_image, (canvas_width, canvas_heigt)) #object_image.save("out/CROPPAD_%d.png"%i) #rotate it object_image = object_image.rotate(objects_rot[i], resample=interp_mode, expand=False) #object_image.save("out/ROTATE_%d.png"%i) #do the translation from the center of the image (do it separately for left and right to avoid interpolation twice) object_image_left = object_image.rotate(0, resample=interp_mode, translate=(objects_dx[i], objects_dy[i])) #object_image_left.save("out/object_image_left_%d.png"%i) #crop back to the sive of the image object_image_left = crop_to_size(object_image_left, (target_w, target_h), canvas_width, canvas_heigt) #FIXME: probably need to be corrupted with noise or something, qs can be deformed i guess object_images_left.append(np.asarray(object_image_left)) #compute diaprity map object_disparity_map = gen_disp_map( object_image_left, objects_disparities[i] ) #TODO:can maybe generate random transforms for depth instead of constant values object_disparity_map_noalpha = gen_disp_map(object_image_left, objects_disparities[i], disp_alpha) object_disparity_map_noaplha_left.append(object_disparity_map_noalpha) object_disparity_map_left.append(object_disparity_map) #Image.fromarray((255*object_disparity_map/max_disp_objects).astype(np.uint8)).save("displ_tmp_%d.png"%i) #translate with disparity (assumes rectified images), not this crops the out of image values object_image_right = object_image.rotate( 0, resample=interp_mode, translate=(objects_dx[i] + objects_disparities[i], objects_dy[i])) #crop back to the sive of the image object_image_right = crop_to_size(object_image_right, (target_w, target_h), canvas_width, canvas_heigt) object_images_right.append(np.asarray(object_image_right)) #object_image_right.save("object_images_right_%d.png"%i) #compute diaprity map object_disparity_map = gen_disp_map( object_image_right, objects_disparities[i] ) #TODO: can maybe generate random transforms for depth instead of constant values object_disparity_map_right.append(object_disparity_map) #Image.fromarray((255*object_disparity_map/max_disp_objects).astype(np.uint8)).save("dispr_tmp_%d.png"%i) if validity_test_function is not None: return gen_scene_stereo_refocus(input_seg_obj_list, input_stereo_left_list, input_stereo_right_list, input_stereo_left_disp_list, input_stereo_right_disp_list, validity_test_function=None) else: #render by clipping synth_left_image, masks_left = clip_images( object_images_left + [background_image_left], object_disparity_map_left + [background_disparity_left]) synth_right_image, masks_right = clip_images( object_images_right + [background_image_right], object_disparity_map_right + [background_disparity_right]) synth_left_disp = clip_disparities(object_disparity_map_noaplha_left + [background_disparity_left]) #refocus on the individual obhect in the right before the background? downsampling_trick_max_kernel_size = 11 synth_left_refocus = refocus_noleak( object_images_left + [background_image_left], object_disparity_map_left + [background_disparity_left], aperture, focus_plane, downsampling_trick_max_kernel_size) #return new disp, lef right images, eventually masks, and occluded masks and disparity (could be usefull to learn disocclusion) return synth_left_image, synth_right_image, synth_left_disp, synth_left_refocus, masks_left, masks_right
def gen_scene_refocus(input_seg_obj_list, input_background_list, input_background_disp_list): assert (len(input_stereo_left_list) == len(input_stereo_right_list) == len(input_stereo_left_disp_list) == len(input_stereo_right_disp_list)) #open background image and disparity background_idx = random.randint(0, len(input_stereo_left_list) - 1) background_image = np.asarray( Image.open(input_stereo_left_list[background_idx])) #resize # background_image = background_image[0:1000,:,:] target_h = background_image.shape[0] target_w = background_image.shape[1] if background_image.shape[2] == 3: background_image = np.concatenate( [background_image, 255 * np.ones([target_h, target_w, 1])], -1) background_disparity = readPFM(input_stereo_left_disp_list[background_idx]) # #background_disparity = background_disparity[0:1000,:] #rescale disparity print(np.amax(background_disparity)) background_disparity = 0.1 * background_disparity print(np.amax(background_disparity)) #pre process backgroud disparity #generate randomness for objects nb_objects = random.randint(min_nb_objects, max_nb_objects) objects_scales = [ random.uniform(min_scale_objects, max_scale_objects) for x in range(nb_objects) ] objects_idxs = [ random.randint(0, len(input_seg_obj_list) - 1) for x in range(nb_objects) ] objects_disparities = [ random.uniform(min_disp_objects, max_disp_objects) for x in range(nb_objects) ] objects_dx = [ round(target_w * random.uniform(min_x_objects, max_x_objects)) for x in range(nb_objects) ] objects_dy = [ round(target_h * random.uniform(min_y_objects, max_y_objects)) for x in range(nb_objects) ] #FIXME: do integer translation? objects_rot = [ random.randint(min_object_rotation, max_object_rotation) for x in range(nb_objects) ] #open and transform the images object_images = [] object_disparity = [] object_disparity_map_noaplha = [] for i in range(nb_objects): object_image = Image.open(input_seg_obj_list[objects_idxs[i]]) #resize image object_image = object_image.resize( (int(object_image.size[0] * objects_scales[i]), int(object_image.size[1] * objects_scales[i]))) #pad to canvas image size so we dont have to worry about corpping accidentatluy things object_image = pad_to_size(object_image, (canvas_width, canvas_heigt)) #object_image.save("out/CROPPAD_%d.png"%i) #rotate it object_image = object_image.rotate(objects_rot[i], resample=interp_mode, expand=False) #object_image.save("out/ROTATE_%d.png"%i) #do the translation from the center of the image (do it separately for left and right to avoid interpolation twice) object_image = object_image.rotate(0, resample=interp_mode, translate=(objects_dx[i], objects_dy[i])) #object_image_left.save("out/object_image_left_%d.png"%i) #crop back to the sive of the image object_image = crop_to_size(object_image, (target_w, target_h), canvas_width, canvas_heigt) #FIXME: probably need to be corrupted with noise or something, qs can be deformed i guess object_images.append(np.asarray(object_image)) #compute disparity map object_disparity_map = gen_disp_map( object_image, objects_disparities[i] ) #TODO:can maybe generate random transforms for depth instead of constant values object_disparity_map_noalpha_tmp = gen_disp_map( object_image, objects_disparities[i], disp_alpha) object_disparity_map_noaplha.append(object_disparity_map_noalpha_tmp) object_disparity.append(object_disparity_map) #render by clipping synth_image, masks = clip_images(object_images + [background_image], object_disparity + [background_disparity]) synth_disp = clip_disparities(object_disparity_map_noaplha + [background_disparity]) #refocus on the individual obhect in the right before the background? downsampling_trick_max_kernel_size = 11 synth_refocus = refocus_noleak(object_images + [background_image], object_disparity + [background_disparity], aperture, focus_plane, downsampling_trick_max_kernel_size) #return new disp, left right images, eventually masks, and occluded masks and disparity (could be usefull to learn disocclusion) return synth_image, synth_disp, synth_refocus, masks
import utils import numpy as np import matplotlib.pyplot as plt from scipy import ndimage from PIL import Image, ImageFilter arr1 = utils.readPFM( 'mid/Adirondack-imperfect/Adirondack-imperfect/disp0.pfm')[0] arr2 = utils.readPFM( 'mid/Adirondack-imperfect/Adirondack-imperfect/disp0y.pfm')[0] arr3 = utils.readPFM( 'mid/Adirondack-imperfect/Adirondack-imperfect/disp1.pfm')[0] arr4 = utils.readPFM( 'mid/Adirondack-imperfect/Adirondack-imperfect/disp1y.pfm')[0] calib = utils.read_calib( 'mid/Adirondack-imperfect/Adirondack-imperfect/calib.txt') b = calib['baseline'] f = calib['cam0'][0, 0] d = calib['doffs'] d1 = b * f / (arr1 + d) d2 = b * f / (arr2 + d) d3 = b * f / (arr3 + d) d4 = b * f / (arr4 + d) d = (d1 - d1.min()) / (d1.max() - d1.min()) d_ = ndimage.maximum_filter(d, size=3, mode='nearest') d_ = ndimage.maximum_filter(d_, size=2, mode='nearest')