def gen_scene_stereo_refocus(input_seg_obj_list,
input_stereo_left_list,
input_stereo_right_list,
input_stereo_left_disp_list,
input_stereo_right_disp_list,
validity_test_function=None):
"""
Generates a random scene and render lef right stereo views, disparity map, objects masks, and refocused image. No check on the final scene is performed.
"""
assert (len(input_stereo_left_list) == len(input_stereo_right_list) ==
len(input_stereo_left_disp_list) ==
len(input_stereo_right_disp_list))
#open background image and disparity
background_idx = random.randint(0, len(input_stereo_left_list) - 1)
background_image_left = np.asarray(
Image.open(input_stereo_left_list[background_idx]))
background_image_right = np.asarray(
Image.open(input_stereo_right_list[background_idx]))
target_h = background_image_left.shape[0]
target_w = background_image_right.shape[1]
if background_image_left.shape[2] == 3:
background_image_left = np.concatenate(
[background_image_left, 255 * np.ones([target_h, target_w, 1])],
-1)
if background_image_right.shape[2] == 3:
background_image_right = np.concatenate(
[background_image_right, 255 * np.ones([target_h, target_w, 1])],
-1)
background_disparity_left = readPFM(
input_stereo_left_disp_list[background_idx])
background_disparity_right = readPFM(
input_stereo_right_disp_list[background_idx])
#generate randomness for objects
nb_objects = random.randint(min_nb_objects, max_nb_objects)
objects_scales = [
random.uniform(min_scale_objects, max_scale_objects)
for x in range(nb_objects)
]
objects_idxs = [
random.randint(0,
len(input_seg_obj_list) - 1) for x in range(nb_objects)
]
objects_disparities = [
random.uniform(min_disp_objects, max_disp_objects)
for x in range(nb_objects)
]
objects_dx = [
round(target_w * random.uniform(min_x_objects, max_x_objects))
for x in range(nb_objects)
]
objects_dy = [
round(target_h * random.uniform(min_y_objects, max_y_objects))
for x in range(nb_objects)
] #FIXME: do integer translation?
objects_rot = [
random.randint(min_object_rotation, max_object_rotation)
for x in range(nb_objects)
]
#open and transform the images
object_images_left = []
object_disparity_map_left = []
object_disparity_map_noaplha_left = []
object_images_right = []
object_disparity_map_right = []
for i in range(nb_objects):
object_image = Image.open(input_seg_obj_list[objects_idxs[i]])
#object_image.save("out/ORIG_%d.png"%i)
#Image.fromarray(np.asarray(object_image)[:,:,3]).save("alpha_%d.png"%i)
#resize image
object_image = object_image.resize(
(int(object_image.size[0] * objects_scales[i]),
int(object_image.size[1] * objects_scales[i])))
#object_image.save("out/RESIZE_%d.png"%i)
#pad to canvas image size so we dont have to worry about corpping accidentatluy things
object_image = pad_to_size(object_image, (canvas_width, canvas_heigt))
#object_image.save("out/CROPPAD_%d.png"%i)
#rotate it
object_image = object_image.rotate(objects_rot[i],
resample=interp_mode,
expand=False)
#object_image.save("out/ROTATE_%d.png"%i)
#do the translation from the center of the image (do it separately for left and right to avoid interpolation twice)
object_image_left = object_image.rotate(0,
resample=interp_mode,
translate=(objects_dx[i],
objects_dy[i]))
#object_image_left.save("out/object_image_left_%d.png"%i)
#crop back to the sive of the image
object_image_left = crop_to_size(object_image_left,
(target_w, target_h), canvas_width,
canvas_heigt)
#FIXME: probably need to be corrupted with noise or something, qs can be deformed i guess
object_images_left.append(np.asarray(object_image_left))
#compute diaprity map
object_disparity_map = gen_disp_map(
object_image_left, objects_disparities[i]
) #TODO:can maybe generate random transforms for depth instead of constant values
object_disparity_map_noalpha = gen_disp_map(object_image_left,
objects_disparities[i],
disp_alpha)
object_disparity_map_noaplha_left.append(object_disparity_map_noalpha)
object_disparity_map_left.append(object_disparity_map)
#Image.fromarray((255*object_disparity_map/max_disp_objects).astype(np.uint8)).save("displ_tmp_%d.png"%i)
#translate with disparity (assumes rectified images), not this crops the out of image values
object_image_right = object_image.rotate(
0,
resample=interp_mode,
translate=(objects_dx[i] + objects_disparities[i], objects_dy[i]))
#crop back to the sive of the image
object_image_right = crop_to_size(object_image_right,
(target_w, target_h), canvas_width,
canvas_heigt)
object_images_right.append(np.asarray(object_image_right))
#object_image_right.save("object_images_right_%d.png"%i)
#compute diaprity map
object_disparity_map = gen_disp_map(
object_image_right, objects_disparities[i]
) #TODO: can maybe generate random transforms for depth instead of constant values
object_disparity_map_right.append(object_disparity_map)
#Image.fromarray((255*object_disparity_map/max_disp_objects).astype(np.uint8)).save("dispr_tmp_%d.png"%i)
if validity_test_function is not None:
return gen_scene_stereo_refocus(input_seg_obj_list,
input_stereo_left_list,
input_stereo_right_list,
input_stereo_left_disp_list,
input_stereo_right_disp_list,
validity_test_function=None)
else:
#render by clipping
synth_left_image, masks_left = clip_images(
object_images_left + [background_image_left],
object_disparity_map_left + [background_disparity_left])
synth_right_image, masks_right = clip_images(
object_images_right + [background_image_right],
object_disparity_map_right + [background_disparity_right])
synth_left_disp = clip_disparities(object_disparity_map_noaplha_left +
[background_disparity_left])
#refocus on the individual obhect in the right before the background?
downsampling_trick_max_kernel_size = 11
synth_left_refocus = refocus_noleak(
object_images_left + [background_image_left],
object_disparity_map_left + [background_disparity_left], aperture,
focus_plane, downsampling_trick_max_kernel_size)
#return new disp, lef right images, eventually masks, and occluded masks and disparity (could be usefull to learn disocclusion)
return synth_left_image, synth_right_image, synth_left_disp, synth_left_refocus, masks_left, masks_right
def gen_scene_refocus(input_seg_obj_list, input_background_list,
input_background_disp_list):
assert (len(input_stereo_left_list) == len(input_stereo_right_list) ==
len(input_stereo_left_disp_list) ==
len(input_stereo_right_disp_list))
#open background image and disparity
background_idx = random.randint(0, len(input_stereo_left_list) - 1)
background_image = np.asarray(
Image.open(input_stereo_left_list[background_idx]))
#resize
# background_image = background_image[0:1000,:,:]
target_h = background_image.shape[0]
target_w = background_image.shape[1]
if background_image.shape[2] == 3:
background_image = np.concatenate(
[background_image, 255 * np.ones([target_h, target_w, 1])], -1)
background_disparity = readPFM(input_stereo_left_disp_list[background_idx])
#
#background_disparity = background_disparity[0:1000,:]
#rescale disparity
print(np.amax(background_disparity))
background_disparity = 0.1 * background_disparity
print(np.amax(background_disparity))
#pre process backgroud disparity
#generate randomness for objects
nb_objects = random.randint(min_nb_objects, max_nb_objects)
objects_scales = [
random.uniform(min_scale_objects, max_scale_objects)
for x in range(nb_objects)
]
objects_idxs = [
random.randint(0,
len(input_seg_obj_list) - 1) for x in range(nb_objects)
]
objects_disparities = [
random.uniform(min_disp_objects, max_disp_objects)
for x in range(nb_objects)
]
objects_dx = [
round(target_w * random.uniform(min_x_objects, max_x_objects))
for x in range(nb_objects)
]
objects_dy = [
round(target_h * random.uniform(min_y_objects, max_y_objects))
for x in range(nb_objects)
] #FIXME: do integer translation?
objects_rot = [
random.randint(min_object_rotation, max_object_rotation)
for x in range(nb_objects)
]
#open and transform the images
object_images = []
object_disparity = []
object_disparity_map_noaplha = []
for i in range(nb_objects):
object_image = Image.open(input_seg_obj_list[objects_idxs[i]])
#resize image
object_image = object_image.resize(
(int(object_image.size[0] * objects_scales[i]),
int(object_image.size[1] * objects_scales[i])))
#pad to canvas image size so we dont have to worry about corpping accidentatluy things
object_image = pad_to_size(object_image, (canvas_width, canvas_heigt))
#object_image.save("out/CROPPAD_%d.png"%i)
#rotate it
object_image = object_image.rotate(objects_rot[i],
resample=interp_mode,
expand=False)
#object_image.save("out/ROTATE_%d.png"%i)
#do the translation from the center of the image (do it separately for left and right to avoid interpolation twice)
object_image = object_image.rotate(0,
resample=interp_mode,
translate=(objects_dx[i],
objects_dy[i]))
#object_image_left.save("out/object_image_left_%d.png"%i)
#crop back to the sive of the image
object_image = crop_to_size(object_image, (target_w, target_h),
canvas_width, canvas_heigt)
#FIXME: probably need to be corrupted with noise or something, qs can be deformed i guess
object_images.append(np.asarray(object_image))
#compute disparity map
object_disparity_map = gen_disp_map(
object_image, objects_disparities[i]
) #TODO:can maybe generate random transforms for depth instead of constant values
object_disparity_map_noalpha_tmp = gen_disp_map(
object_image, objects_disparities[i], disp_alpha)
object_disparity_map_noaplha.append(object_disparity_map_noalpha_tmp)
object_disparity.append(object_disparity_map)
#render by clipping
synth_image, masks = clip_images(object_images + [background_image],
object_disparity + [background_disparity])
synth_disp = clip_disparities(object_disparity_map_noaplha +
[background_disparity])
#refocus on the individual obhect in the right before the background?
downsampling_trick_max_kernel_size = 11
synth_refocus = refocus_noleak(object_images + [background_image],
object_disparity + [background_disparity],
aperture, focus_plane,
downsampling_trick_max_kernel_size)
#return new disp, left right images, eventually masks, and occluded masks and disparity (could be usefull to learn disocclusion)
return synth_image, synth_disp, synth_refocus, masks