def refnet_blur_baseline(left_image, right_image, target_disparities, blur_magnitude,
is_training=True, stop_grads = True,
min_disp = 0, max_disp = 300, downsampling_trick_max_kernel_size=11,
from_stage = "disparity_map", differenciable=False
):
"""
Model that refocuses at the coarsest level of the cost volume and then tryies
to upsample with residual learning the blured image.
left_image, right_image: placeholder for BxNxMx3 images
target_disparities: single or list of focus planes to refocus with
blur_magnitude: virtual aperture
min_disp max_disp = 300: min and max possible disparity (determine kernel sizes)
downsampling_trick_max_kernel_size: set a maximum kernel size to be used, if overflowed, downsampling will be used.
from_stage: uses the disparity map at full resolution or the cost volume to perform refocusing.
Returns: a list of refocused images, the disparity or cost volume placeholders.
"""
#FIXME: bug when is_training=False
disparity, intermediate_steps = stereonet(left_image, right_image, is_training=True)#is_training=(not stop_grads))
cost_volume = intermediate_steps["cost_volume_left_view"]
refocus_images = []
intermediate_result = []
if from_stage == "disparity_map":
if stop_grads:
disparity = tf.stop_gradient(disparity)
for target_disparity in target_disparities:
refocus_image = layered_bluring(left_image, disparity, target_disparity,blur_magnitude,
min_disp, max_disp,downsampling_trick_max_kernel_size,
differenciable=differenciable)
refocus_images.append(refocus_image)
return refocus_images, disparity, intermediate_steps
elif from_stage == "cost_volume":
if stop_grads:
cost_volume = tf.stop_gradient(cost_volume)
#need to normalise cost volume, note: can play with beta
#note sofmtin turn it into a confidence volume
beta = 1
conf_volume = tf.nn.softmax(-beta*cost_volume, dim=-1)
#this if given by stereonet architecture
disparity_range = np.arange(1,18+1)*8#FIXME: see ben for disp=0
refocus_images = []
for target_disparity in target_disparities:
refocus_image = layered_bluring_from_cost_volume(left_image, conf_volume, disparity_range,
target_disparity, blur_magnitude,
downsampling_trick_max_kernel_size,
differenciable=differenciable)
refocus_images.append(refocus_image)
return refocus_images, conf_volume, intermediate_steps
else:
raise BaseException("Stage type not understood. Needs to be 'disparity_map' or 'cost_volume' not '%s'"%from_stage)
def do_blur(image, disparity_map, output_folder, focus_plane, aperture,
pyramidal_conv, disparity_range, verbose):
#defines verbose level
if verbose >= 2:
tf.logging.set_verbosity(tf.logging.INFO)
elif verbose >= 1:
tf.logging.set_verbosity(tf.logging.WARN)
else:
tf.logging.set_verbosity(tf.logging.ERROR)
#remove verbose bits from tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
#Opening images
tf.logging.info("Opening files")
image = np.expand_dims(Image.open(image), 0).astype(np.float32) / 255.0
image = image[:, :, :, 0:3]
if disparity_map[-3:] == "npy":
disp_map = np.expand_dims(np.expand_dims(np.load(disparity_map), 0),
-1)
elif disparity_map[-3:] == "png":
tf.logging.warn(
"Loading disparty map from nor;alised png. Disparity values between 0 and 1"
)
disp_map = np.expand_dims(Image.open(disparity_map), 0).astype(
np.float32) / 255.0
disp_map = np.expand_dims(disp_map[:, :, :, 0], -1)
else:
raise BaseException("Disparity map format unsupported yet")
h, w = image.shape[1:3]
assert (image.shape[1:3] == disp_map.shape[1:3],
"Disparity map and image of different size.")
#making output folder if needed
if not os.path.exists(output_folder):
os.makedirs(output_folder)
#Building the graph
tf.logging.info("Making graph for %d focal planes" % len(focus_plane))
img_ph = tf.placeholder(tf.float32, shape=[1, h, w, 3])
disp_ph = tf.placeholder(tf.float32, shape=[1, h, w, 1])
output_ph = []
for f in focus_plane:
output_ph.append(
layered_bluring(img_ph,
disp_ph,
target_disparity=f,
blur_magnitude=aperture,
min_disp=disparity_range[0],
max_disp=disparity_range[1],
downsampling_trick_max_kernel_size=pyramidal_conv,
differenciable=False))
#Runs the thing
with tf.Session() as sess:
tf.logging.info("Runing refocusing")
refocus_image = sess.run(output_ph,
feed_dict={
img_ph: image,
disp_ph: disp_map
})
tf.logging.info("Done")
tf.logging.info("Saving to " + output_folder +
"/refocused_image_[focus_plane].png")
for i in range(len(focus_plane)):
f = focus_plane[i]
Image.fromarray((refocus_image[i][0, :, :, :] * 255).astype(
np.uint8)).save(output_folder + "/refocused_image_%f.png" % f)
def refnet_blur_refinement(left_image,
right_image,
target_disparity,
blur_magnitude,
min_disp=0,
max_disp=300,
is_training=True,
stop_grads=True,
from_scale=1):
"""
Model that refocuses at the coarsest level of the cost volume and then tryies
to upsample with residual learning the blured image.
"""
heigt = int(left_image.shape[1])
width = int(left_image.shape[2])
#Get cost volume from the first stage of stereonet
_, intermediate_steps = stereonet(
left_image, right_image,
is_training=True) #FIXME: not working with true
disparity_1_2 = intermediate_steps["disparity_map_1_2"]
disparity_1_4 = intermediate_steps["disparity_map_1_4"]
disparity_1_8 = intermediate_steps["disparity_map_1_8"]
with tf.variable_scope("blur_upsampling"):
if from_scale == 3:
lowres_disparity = disparity_1_8
elif from_scale == 2:
lowres_disparity = disparity_1_4
elif from_scale == 1:
lowres_disparity = disparity_1_2
else:
raise BaseException("Scale id not recognised")
heigt_lowres = int(lowres_disparity.shape[1])
width_lowres = int(lowres_disparity.shape[2])
if stop_grads:
lowres_disparity = tf.stop_gradient(lowres_disparity)
target_disparity = target_disparity / (2**from_scale
) #defined by deep stereo
min_disp = min_disp / (2**from_scale)
max_disp = max_disp / (2**from_scale)
print(
"At low res focus is %f and magnitude %f min max disparity %f %f" %
(target_disparity, blur_magnitude, min_disp, max_disp))
#refocus at low res direclty from the cost volume
left_image_lowres = tf.image.resize_images(left_image,
size=(heigt_lowres,
width_lowres),
align_corners=True)
refocus_lowres = layered_bluring(
left_image_lowres,
lowres_disparity,
target_disparity,
blur_magnitude,
min_disp,
max_disp,
downsampling_trick_max_kernel_size=None, #No need trick
differenciable=True)
initial_refocus = refocus_lowres
#upsampling with residual learning
if from_scale == 3:
tf.logging.info("Upsampling from scale 1/8 to 1/4")
#get downsampled/upsampled images
left_image_1_4 = tf.image.resize_images(left_image,
size=(int(heigt / 4),
int(width / 4)),
align_corners=True)
#scale 1/4
refocus_1_4_coarse = tf.image.resize_images(refocus_lowres,
size=(int(heigt / 4),
int(width / 4)),
align_corners=True)
refocus_lowres = _blur_image_refinement_net(
left_image_1_4,
refocus_1_4_coarse,
scale_name="1_4",
is_training=is_training)
if from_scale >= 2:
tf.logging.info("Upsampling from scale 1/4 to 1/2")
left_image_1_2 = tf.image.resize_images(left_image,
size=(int(heigt / 2),
int(width / 2)),
align_corners=True)
#scale 1/2
refocus_1_2_coarse = tf.image.resize_images(refocus_lowres,
size=(int(heigt / 2),
int(width / 2)),
align_corners=True)
refocus_lowres = _blur_image_refinement_net(
left_image_1_2,
refocus_1_2_coarse,
scale_name="1_2",
is_training=is_training)
if from_scale >= 1:
tf.logging.info("Upsampling from scale 1/2 to 1/1")
#full scale
refocus_1_1_coarse = tf.image.resize_images(refocus_lowres,
size=(int(heigt),
int(width)),
align_corners=True)
refocus_1_1 = _blur_image_refinement_net(left_image,
refocus_1_1_coarse,
scale_name="1_1",
is_training=is_training)
return refocus_1_1, initial_refocus, lowres_disparity