def read_dir(dir_path, scale=False):
disp0_path = path.join(dir_path, 'disp0.pfm')
disp1_path = path.join(dir_path, 'disp1.pfm')
im0_path = path.join(dir_path, 'im0.png')
im1_path = path.join(dir_path, 'im1.png')
im1E_path = path.join(dir_path, 'im1E.png')
im1L_path = path.join(dir_path, 'im1L.png')
calib_path = path.join(dir_path, 'calib.txt')
print('Loading disparities')
disp0 = utils.readPFM(disp0_path)[0]
disp1 = utils.readPFM(disp1_path)[0]
print('Loading images')
im0 = np.array(Image.open(im0_path).convert('RGB'))
im1 = np.array(Image.open(im1_path).convert('RGB'))
im1E = np.array(Image.open(im1E_path).convert('RGB'))
im1L = np.array(Image.open(im1L_path).convert('RGB'))
if scale:
print('Scaling Images')
im0 = imresize(im0, IMSIZE + im0.shape[2:])
im1 = imresize(im1, IMSIZE + im1.shape[2:])
im1E = imresize(im1E, IMSIZE + im1E.shape[2:])
im1L = imresize(im1L, IMSIZE + im1L.shape[2:])
print('Loading calib')
calib = utils.read_calib(calib_path)
b = calib['baseline']
f = calib['cam0'][0, 0]
d = calib['doffs']
print('Creating depth maps')
d0 = b * f / (disp0 + d)
d1 = b * f / (disp1 + d)
d0 = preprocess(d0)
d1 = preprocess(d1)
if scale:
print('Scaling depth maps')
d0 = imresize(d0, IMSIZE)[..., None]
d1 = imresize(d1, IMSIZE)[..., None]
return im0, im1, im1E, im1L, d0, d1
def load_batch(self, left, right, labels, is_training):
batch_left = []
batch_right = []
batch_label = []
for x, y, z in zip(left, right, labels):
# print(x)
if is_training:
crop_x = random.randint(
0, self.img_height - 1 - self.patch_size[0])
crop_y = random.randint(
0, self.img_width - 1 - self.patch_size[1])
else:
crop_x = (self.img_height - 1 - self.patch_size[0]) // 2
crop_y = (self.img_width - 1 - self.patch_size[1]) // 2
x = cv2.imread(x)
assert x.shape[:2] == (self.img_height, self.img_width)
x = cv2.cvtColor(x, cv2.COLOR_BGR2RGB)
x = x[crop_x:crop_x + self.patch_size[0],
crop_y:crop_y + self.patch_size[1], :]
x = utils.mean_std(x)
batch_left.append(x)
y = cv2.imread(y)
y = cv2.cvtColor(y, cv2.COLOR_BGR2RGB)
y = y[crop_x:crop_x + self.patch_size[0],
crop_y:crop_y + self.patch_size[1], :]
y = utils.mean_std(y)
batch_right.append(y)
z = readPFM(z)
# z = cv2.cvtColor(z, cv2.COLOR_BGR2GRAY)
z = z[crop_x:crop_x + self.patch_size[0],
crop_y:crop_y + self.patch_size[1]]
z[z > (self.max_disp - 1)] = self.max_disp - 1
batch_label.append(z)
return batch_left, batch_right, batch_label
def gen_scene_stereo_refocus(input_seg_obj_list,
input_stereo_left_list,
input_stereo_right_list,
input_stereo_left_disp_list,
input_stereo_right_disp_list,
validity_test_function=None):
"""
Generates a random scene and render lef right stereo views, disparity map, objects masks, and refocused image. No check on the final scene is performed.
"""
assert (len(input_stereo_left_list) == len(input_stereo_right_list) ==
len(input_stereo_left_disp_list) ==
len(input_stereo_right_disp_list))
#open background image and disparity
background_idx = random.randint(0, len(input_stereo_left_list) - 1)
background_image_left = np.asarray(
Image.open(input_stereo_left_list[background_idx]))
background_image_right = np.asarray(
Image.open(input_stereo_right_list[background_idx]))
target_h = background_image_left.shape[0]
target_w = background_image_right.shape[1]
if background_image_left.shape[2] == 3:
background_image_left = np.concatenate(
[background_image_left, 255 * np.ones([target_h, target_w, 1])],
-1)
if background_image_right.shape[2] == 3:
background_image_right = np.concatenate(
[background_image_right, 255 * np.ones([target_h, target_w, 1])],
-1)
background_disparity_left = readPFM(
input_stereo_left_disp_list[background_idx])
background_disparity_right = readPFM(
input_stereo_right_disp_list[background_idx])
#generate randomness for objects
nb_objects = random.randint(min_nb_objects, max_nb_objects)
objects_scales = [
random.uniform(min_scale_objects, max_scale_objects)
for x in range(nb_objects)
]
objects_idxs = [
random.randint(0,
len(input_seg_obj_list) - 1) for x in range(nb_objects)
]
objects_disparities = [
random.uniform(min_disp_objects, max_disp_objects)
for x in range(nb_objects)
]
objects_dx = [
round(target_w * random.uniform(min_x_objects, max_x_objects))
for x in range(nb_objects)
]
objects_dy = [
round(target_h * random.uniform(min_y_objects, max_y_objects))
for x in range(nb_objects)
] #FIXME: do integer translation?
objects_rot = [
random.randint(min_object_rotation, max_object_rotation)
for x in range(nb_objects)
]
#open and transform the images
object_images_left = []
object_disparity_map_left = []
object_disparity_map_noaplha_left = []
object_images_right = []
object_disparity_map_right = []
for i in range(nb_objects):
object_image = Image.open(input_seg_obj_list[objects_idxs[i]])
#object_image.save("out/ORIG_%d.png"%i)
#Image.fromarray(np.asarray(object_image)[:,:,3]).save("alpha_%d.png"%i)
#resize image
object_image = object_image.resize(
(int(object_image.size[0] * objects_scales[i]),
int(object_image.size[1] * objects_scales[i])))
#object_image.save("out/RESIZE_%d.png"%i)
#pad to canvas image size so we dont have to worry about corpping accidentatluy things
object_image = pad_to_size(object_image, (canvas_width, canvas_heigt))
#object_image.save("out/CROPPAD_%d.png"%i)
#rotate it
object_image = object_image.rotate(objects_rot[i],
resample=interp_mode,
expand=False)
#object_image.save("out/ROTATE_%d.png"%i)
#do the translation from the center of the image (do it separately for left and right to avoid interpolation twice)
object_image_left = object_image.rotate(0,
resample=interp_mode,
translate=(objects_dx[i],
objects_dy[i]))
#object_image_left.save("out/object_image_left_%d.png"%i)
#crop back to the sive of the image
object_image_left = crop_to_size(object_image_left,
(target_w, target_h), canvas_width,
canvas_heigt)
#FIXME: probably need to be corrupted with noise or something, qs can be deformed i guess
object_images_left.append(np.asarray(object_image_left))
#compute diaprity map
object_disparity_map = gen_disp_map(
object_image_left, objects_disparities[i]
) #TODO:can maybe generate random transforms for depth instead of constant values
object_disparity_map_noalpha = gen_disp_map(object_image_left,
objects_disparities[i],
disp_alpha)
object_disparity_map_noaplha_left.append(object_disparity_map_noalpha)
object_disparity_map_left.append(object_disparity_map)
#Image.fromarray((255*object_disparity_map/max_disp_objects).astype(np.uint8)).save("displ_tmp_%d.png"%i)
#translate with disparity (assumes rectified images), not this crops the out of image values
object_image_right = object_image.rotate(
0,
resample=interp_mode,
translate=(objects_dx[i] + objects_disparities[i], objects_dy[i]))
#crop back to the sive of the image
object_image_right = crop_to_size(object_image_right,
(target_w, target_h), canvas_width,
canvas_heigt)
object_images_right.append(np.asarray(object_image_right))
#object_image_right.save("object_images_right_%d.png"%i)
#compute diaprity map
object_disparity_map = gen_disp_map(
object_image_right, objects_disparities[i]
) #TODO: can maybe generate random transforms for depth instead of constant values
object_disparity_map_right.append(object_disparity_map)
#Image.fromarray((255*object_disparity_map/max_disp_objects).astype(np.uint8)).save("dispr_tmp_%d.png"%i)
if validity_test_function is not None:
return gen_scene_stereo_refocus(input_seg_obj_list,
input_stereo_left_list,
input_stereo_right_list,
input_stereo_left_disp_list,
input_stereo_right_disp_list,
validity_test_function=None)
else:
#render by clipping
synth_left_image, masks_left = clip_images(
object_images_left + [background_image_left],
object_disparity_map_left + [background_disparity_left])
synth_right_image, masks_right = clip_images(
object_images_right + [background_image_right],
object_disparity_map_right + [background_disparity_right])
synth_left_disp = clip_disparities(object_disparity_map_noaplha_left +
[background_disparity_left])
#refocus on the individual obhect in the right before the background?
downsampling_trick_max_kernel_size = 11
synth_left_refocus = refocus_noleak(
object_images_left + [background_image_left],
object_disparity_map_left + [background_disparity_left], aperture,
focus_plane, downsampling_trick_max_kernel_size)
#return new disp, lef right images, eventually masks, and occluded masks and disparity (could be usefull to learn disocclusion)
return synth_left_image, synth_right_image, synth_left_disp, synth_left_refocus, masks_left, masks_right
def gen_scene_refocus(input_seg_obj_list, input_background_list,
input_background_disp_list):
assert (len(input_stereo_left_list) == len(input_stereo_right_list) ==
len(input_stereo_left_disp_list) ==
len(input_stereo_right_disp_list))
#open background image and disparity
background_idx = random.randint(0, len(input_stereo_left_list) - 1)
background_image = np.asarray(
Image.open(input_stereo_left_list[background_idx]))
#resize
# background_image = background_image[0:1000,:,:]
target_h = background_image.shape[0]
target_w = background_image.shape[1]
if background_image.shape[2] == 3:
background_image = np.concatenate(
[background_image, 255 * np.ones([target_h, target_w, 1])], -1)
background_disparity = readPFM(input_stereo_left_disp_list[background_idx])
#
#background_disparity = background_disparity[0:1000,:]
#rescale disparity
print(np.amax(background_disparity))
background_disparity = 0.1 * background_disparity
print(np.amax(background_disparity))
#pre process backgroud disparity
#generate randomness for objects
nb_objects = random.randint(min_nb_objects, max_nb_objects)
objects_scales = [
random.uniform(min_scale_objects, max_scale_objects)
for x in range(nb_objects)
]
objects_idxs = [
random.randint(0,
len(input_seg_obj_list) - 1) for x in range(nb_objects)
]
objects_disparities = [
random.uniform(min_disp_objects, max_disp_objects)
for x in range(nb_objects)
]
objects_dx = [
round(target_w * random.uniform(min_x_objects, max_x_objects))
for x in range(nb_objects)
]
objects_dy = [
round(target_h * random.uniform(min_y_objects, max_y_objects))
for x in range(nb_objects)
] #FIXME: do integer translation?
objects_rot = [
random.randint(min_object_rotation, max_object_rotation)
for x in range(nb_objects)
]
#open and transform the images
object_images = []
object_disparity = []
object_disparity_map_noaplha = []
for i in range(nb_objects):
object_image = Image.open(input_seg_obj_list[objects_idxs[i]])
#resize image
object_image = object_image.resize(
(int(object_image.size[0] * objects_scales[i]),
int(object_image.size[1] * objects_scales[i])))
#pad to canvas image size so we dont have to worry about corpping accidentatluy things
object_image = pad_to_size(object_image, (canvas_width, canvas_heigt))
#object_image.save("out/CROPPAD_%d.png"%i)
#rotate it
object_image = object_image.rotate(objects_rot[i],
resample=interp_mode,
expand=False)
#object_image.save("out/ROTATE_%d.png"%i)
#do the translation from the center of the image (do it separately for left and right to avoid interpolation twice)
object_image = object_image.rotate(0,
resample=interp_mode,
translate=(objects_dx[i],
objects_dy[i]))
#object_image_left.save("out/object_image_left_%d.png"%i)
#crop back to the sive of the image
object_image = crop_to_size(object_image, (target_w, target_h),
canvas_width, canvas_heigt)
#FIXME: probably need to be corrupted with noise or something, qs can be deformed i guess
object_images.append(np.asarray(object_image))
#compute disparity map
object_disparity_map = gen_disp_map(
object_image, objects_disparities[i]
) #TODO:can maybe generate random transforms for depth instead of constant values
object_disparity_map_noalpha_tmp = gen_disp_map(
object_image, objects_disparities[i], disp_alpha)
object_disparity_map_noaplha.append(object_disparity_map_noalpha_tmp)
object_disparity.append(object_disparity_map)
#render by clipping
synth_image, masks = clip_images(object_images + [background_image],
object_disparity + [background_disparity])
synth_disp = clip_disparities(object_disparity_map_noaplha +
[background_disparity])
#refocus on the individual obhect in the right before the background?
downsampling_trick_max_kernel_size = 11
synth_refocus = refocus_noleak(object_images + [background_image],
object_disparity + [background_disparity],
aperture, focus_plane,
downsampling_trick_max_kernel_size)
#return new disp, left right images, eventually masks, and occluded masks and disparity (could be usefull to learn disocclusion)
return synth_image, synth_disp, synth_refocus, masks
import utils
import numpy as np
import matplotlib.pyplot as plt
from scipy import ndimage
from PIL import Image, ImageFilter
arr1 = utils.readPFM(
'mid/Adirondack-imperfect/Adirondack-imperfect/disp0.pfm')[0]
arr2 = utils.readPFM(
'mid/Adirondack-imperfect/Adirondack-imperfect/disp0y.pfm')[0]
arr3 = utils.readPFM(
'mid/Adirondack-imperfect/Adirondack-imperfect/disp1.pfm')[0]
arr4 = utils.readPFM(
'mid/Adirondack-imperfect/Adirondack-imperfect/disp1y.pfm')[0]
calib = utils.read_calib(
'mid/Adirondack-imperfect/Adirondack-imperfect/calib.txt')
b = calib['baseline']
f = calib['cam0'][0, 0]
d = calib['doffs']
d1 = b * f / (arr1 + d)
d2 = b * f / (arr2 + d)
d3 = b * f / (arr3 + d)
d4 = b * f / (arr4 + d)
d = (d1 - d1.min()) / (d1.max() - d1.min())
d_ = ndimage.maximum_filter(d, size=3, mode='nearest')
d_ = ndimage.maximum_filter(d_, size=2, mode='nearest')
