def refnet_blur_baseline(left_image, right_image, target_disparities, blur_magnitude, is_training=True, stop_grads = True, min_disp = 0, max_disp = 300, downsampling_trick_max_kernel_size=11, from_stage = "disparity_map", differenciable=False ): """ Model that refocuses at the coarsest level of the cost volume and then tryies to upsample with residual learning the blured image. left_image, right_image: placeholder for BxNxMx3 images target_disparities: single or list of focus planes to refocus with blur_magnitude: virtual aperture min_disp max_disp = 300: min and max possible disparity (determine kernel sizes) downsampling_trick_max_kernel_size: set a maximum kernel size to be used, if overflowed, downsampling will be used. from_stage: uses the disparity map at full resolution or the cost volume to perform refocusing. Returns: a list of refocused images, the disparity or cost volume placeholders. """ #FIXME: bug when is_training=False disparity, intermediate_steps = stereonet(left_image, right_image, is_training=True)#is_training=(not stop_grads)) cost_volume = intermediate_steps["cost_volume_left_view"] refocus_images = [] intermediate_result = [] if from_stage == "disparity_map": if stop_grads: disparity = tf.stop_gradient(disparity) for target_disparity in target_disparities: refocus_image = layered_bluring(left_image, disparity, target_disparity,blur_magnitude, min_disp, max_disp,downsampling_trick_max_kernel_size, differenciable=differenciable) refocus_images.append(refocus_image) return refocus_images, disparity, intermediate_steps elif from_stage == "cost_volume": if stop_grads: cost_volume = tf.stop_gradient(cost_volume) #need to normalise cost volume, note: can play with beta #note sofmtin turn it into a confidence volume beta = 1 conf_volume = tf.nn.softmax(-beta*cost_volume, dim=-1) #this if given by stereonet architecture disparity_range = np.arange(1,18+1)*8#FIXME: see ben for disp=0 refocus_images = [] for target_disparity in target_disparities: refocus_image = layered_bluring_from_cost_volume(left_image, conf_volume, disparity_range, target_disparity, blur_magnitude, downsampling_trick_max_kernel_size, differenciable=differenciable) refocus_images.append(refocus_image) return refocus_images, conf_volume, intermediate_steps else: raise BaseException("Stage type not understood. Needs to be 'disparity_map' or 'cost_volume' not '%s'"%from_stage)
def do_blur(image, disparity_map, output_folder, focus_plane, aperture, pyramidal_conv, disparity_range, verbose): #defines verbose level if verbose >= 2: tf.logging.set_verbosity(tf.logging.INFO) elif verbose >= 1: tf.logging.set_verbosity(tf.logging.WARN) else: tf.logging.set_verbosity(tf.logging.ERROR) #remove verbose bits from tf os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' #Opening images tf.logging.info("Opening files") image = np.expand_dims(Image.open(image), 0).astype(np.float32) / 255.0 image = image[:, :, :, 0:3] if disparity_map[-3:] == "npy": disp_map = np.expand_dims(np.expand_dims(np.load(disparity_map), 0), -1) elif disparity_map[-3:] == "png": tf.logging.warn( "Loading disparty map from nor;alised png. Disparity values between 0 and 1" ) disp_map = np.expand_dims(Image.open(disparity_map), 0).astype( np.float32) / 255.0 disp_map = np.expand_dims(disp_map[:, :, :, 0], -1) else: raise BaseException("Disparity map format unsupported yet") h, w = image.shape[1:3] assert (image.shape[1:3] == disp_map.shape[1:3], "Disparity map and image of different size.") #making output folder if needed if not os.path.exists(output_folder): os.makedirs(output_folder) #Building the graph tf.logging.info("Making graph for %d focal planes" % len(focus_plane)) img_ph = tf.placeholder(tf.float32, shape=[1, h, w, 3]) disp_ph = tf.placeholder(tf.float32, shape=[1, h, w, 1]) output_ph = [] for f in focus_plane: output_ph.append( layered_bluring(img_ph, disp_ph, target_disparity=f, blur_magnitude=aperture, min_disp=disparity_range[0], max_disp=disparity_range[1], downsampling_trick_max_kernel_size=pyramidal_conv, differenciable=False)) #Runs the thing with tf.Session() as sess: tf.logging.info("Runing refocusing") refocus_image = sess.run(output_ph, feed_dict={ img_ph: image, disp_ph: disp_map }) tf.logging.info("Done") tf.logging.info("Saving to " + output_folder + "/refocused_image_[focus_plane].png") for i in range(len(focus_plane)): f = focus_plane[i] Image.fromarray((refocus_image[i][0, :, :, :] * 255).astype( np.uint8)).save(output_folder + "/refocused_image_%f.png" % f)
def refnet_blur_refinement(left_image, right_image, target_disparity, blur_magnitude, min_disp=0, max_disp=300, is_training=True, stop_grads=True, from_scale=1): """ Model that refocuses at the coarsest level of the cost volume and then tryies to upsample with residual learning the blured image. """ heigt = int(left_image.shape[1]) width = int(left_image.shape[2]) #Get cost volume from the first stage of stereonet _, intermediate_steps = stereonet( left_image, right_image, is_training=True) #FIXME: not working with true disparity_1_2 = intermediate_steps["disparity_map_1_2"] disparity_1_4 = intermediate_steps["disparity_map_1_4"] disparity_1_8 = intermediate_steps["disparity_map_1_8"] with tf.variable_scope("blur_upsampling"): if from_scale == 3: lowres_disparity = disparity_1_8 elif from_scale == 2: lowres_disparity = disparity_1_4 elif from_scale == 1: lowres_disparity = disparity_1_2 else: raise BaseException("Scale id not recognised") heigt_lowres = int(lowres_disparity.shape[1]) width_lowres = int(lowres_disparity.shape[2]) if stop_grads: lowres_disparity = tf.stop_gradient(lowres_disparity) target_disparity = target_disparity / (2**from_scale ) #defined by deep stereo min_disp = min_disp / (2**from_scale) max_disp = max_disp / (2**from_scale) print( "At low res focus is %f and magnitude %f min max disparity %f %f" % (target_disparity, blur_magnitude, min_disp, max_disp)) #refocus at low res direclty from the cost volume left_image_lowres = tf.image.resize_images(left_image, size=(heigt_lowres, width_lowres), align_corners=True) refocus_lowres = layered_bluring( left_image_lowres, lowres_disparity, target_disparity, blur_magnitude, min_disp, max_disp, downsampling_trick_max_kernel_size=None, #No need trick differenciable=True) initial_refocus = refocus_lowres #upsampling with residual learning if from_scale == 3: tf.logging.info("Upsampling from scale 1/8 to 1/4") #get downsampled/upsampled images left_image_1_4 = tf.image.resize_images(left_image, size=(int(heigt / 4), int(width / 4)), align_corners=True) #scale 1/4 refocus_1_4_coarse = tf.image.resize_images(refocus_lowres, size=(int(heigt / 4), int(width / 4)), align_corners=True) refocus_lowres = _blur_image_refinement_net( left_image_1_4, refocus_1_4_coarse, scale_name="1_4", is_training=is_training) if from_scale >= 2: tf.logging.info("Upsampling from scale 1/4 to 1/2") left_image_1_2 = tf.image.resize_images(left_image, size=(int(heigt / 2), int(width / 2)), align_corners=True) #scale 1/2 refocus_1_2_coarse = tf.image.resize_images(refocus_lowres, size=(int(heigt / 2), int(width / 2)), align_corners=True) refocus_lowres = _blur_image_refinement_net( left_image_1_2, refocus_1_2_coarse, scale_name="1_2", is_training=is_training) if from_scale >= 1: tf.logging.info("Upsampling from scale 1/2 to 1/1") #full scale refocus_1_1_coarse = tf.image.resize_images(refocus_lowres, size=(int(heigt), int(width)), align_corners=True) refocus_1_1 = _blur_image_refinement_net(left_image, refocus_1_1_coarse, scale_name="1_1", is_training=is_training) return refocus_1_1, initial_refocus, lowres_disparity