def check_interpolation_correctness(self, shape, image_type, flow_type, num_probes=5): """Interpolate, and then assert correctness for a few query locations.""" image, flows = self.get_image_and_flow_placeholders( shape, image_type, flow_type) interp = dense_image_warp.dense_image_warp(image, flows) low_precision = image_type == 'float16' or flow_type == 'float16' with self.cached_session() as sess: rand_image, rand_flows = self.get_random_image_and_flows( shape, image_type, flow_type) pred_interpolation = sess.run(interp, feed_dict={ image: rand_image, flows: rand_flows }) for _ in range(num_probes): batch_index = np.random.randint(0, shape[0]) y_index = np.random.randint(0, shape[1]) x_index = np.random.randint(0, shape[2]) self.assert_correct_interpolation_value( rand_image, rand_flows, pred_interpolation, batch_index, y_index, x_index, low_precision=low_precision)
def test_gradients_exist(self): """Check that backprop can run. The correctness of the gradients is assumed, since the forward propagation is tested to be correct and we only use built-in tf ops. However, we perform a simple test to make sure that backprop can actually run. We treat the flows as a tf.Variable and optimize them to minimize the difference between the interpolated image and the input image. """ batch_size, height, width, numchannels = [4, 5, 6, 7] image_shape = [batch_size, height, width, numchannels] image = random_ops.random_normal(image_shape) flow_shape = [batch_size, height, width, 2] init_flows = np.float32(np.random.normal(size=flow_shape) * 0.25) flows = variables.Variable(init_flows) interp = dense_image_warp.dense_image_warp(image, flows) loss = math_ops.reduce_mean(math_ops.square(interp - image)) optimizer = adam.AdamOptimizer(1.0) grad = gradients.gradients(loss, [flows]) opt_func = optimizer.apply_gradients(zip(grad, [flows])) init_op = variables.global_variables_initializer() with self.cached_session() as sess: sess.run(init_op) for _ in range(10): sess.run(opt_func)
def test_gradients_exist(self): """Check that backprop can run. The correctness of the gradients is assumed, since the forward propagation is tested to be correct and we only use built-in tf ops. However, we perform a simple test to make sure that backprop can actually run. We treat the flows as a tf.Variable and optimize them to minimize the difference between the interpolated image and the input image. """ batch_size, height, width, numchannels = [4, 5, 6, 7] image_shape = [batch_size, height, width, numchannels] image = random_ops.random_normal(image_shape) flow_shape = [batch_size, height, width, 2] init_flows = np.float32(np.random.normal(size=flow_shape) * 0.25) flows = variables.Variable(init_flows) interp = dense_image_warp.dense_image_warp(image, flows) loss = math_ops.reduce_mean(math_ops.square(interp - image)) optimizer = adam.AdamOptimizer(1.0) grad = gradients.gradients(loss, [flows]) opt_func = optimizer.apply_gradients(zip(grad, [flows])) init_op = variables.global_variables_initializer() with self.test_session() as sess: sess.run(init_op) for _ in range(10): sess.run(opt_func)
def check_interpolation_correctness(self, shape, image_type, flow_type, num_probes=5): """Interpolate, and then assert correctness for a few query locations.""" image, flows = self.get_image_and_flow_placeholders(shape, image_type, flow_type) interp = dense_image_warp.dense_image_warp(image, flows) low_precision = image_type == 'float16' or flow_type == 'float16' with self.test_session() as sess: rand_image, rand_flows = self.get_random_image_and_flows( shape, image_type, flow_type) pred_interpolation = sess.run( interp, feed_dict={ image: rand_image, flows: rand_flows }) for _ in range(num_probes): batch_index = np.random.randint(0, shape[0]) y_index = np.random.randint(0, shape[1]) x_index = np.random.randint(0, shape[2]) self.assert_correct_interpolation_value( rand_image, rand_flows, pred_interpolation, batch_index, y_index, x_index, low_precision=low_precision)
def check_zero_flow_correctness(self, shape, image_type, flow_type): """Assert using zero flows doesn't change the input image.""" image, flows = self.get_image_and_flow_placeholders( shape, image_type, flow_type) interp = dense_image_warp.dense_image_warp(image, flows) with self.cached_session() as sess: rand_image, rand_flows = self.get_random_image_and_flows( shape, image_type, flow_type) rand_flows *= 0 predicted_interpolation = sess.run(interp, feed_dict={ image: rand_image, flows: rand_flows }) self.assertAllClose(rand_image, predicted_interpolation)
def check_zero_flow_correctness(self, shape, image_type, flow_type): """Assert using zero flows doesn't change the input image.""" image, flows = self.get_image_and_flow_placeholders(shape, image_type, flow_type) interp = dense_image_warp.dense_image_warp(image, flows) with self.test_session() as sess: rand_image, rand_flows = self.get_random_image_and_flows( shape, image_type, flow_type) rand_flows *= 0 predicted_interpolation = sess.run( interp, feed_dict={ image: rand_image, flows: rand_flows }) self.assertAllClose(rand_image, predicted_interpolation)
def sparse_image_warp(image, source_control_point_locations, dest_control_point_locations, interpolation_order=2, regularization_weight=0.0, num_boundary_points=0, name='sparse_image_warp'): """Image warping using correspondences between sparse control points. Apply a non-linear warp to the image, where the warp is specified by the source and destination locations of a (potentially small) number of control points. First, we use a polyharmonic spline (`tf.contrib.image.interpolate_spline`) to interpolate the displacements between the corresponding control points to a dense flow field. Then, we warp the image using this dense flow field (`tf.contrib.image.dense_image_warp`). Let t index our control points. For regularization_weight=0, we have: warped_image[b, dest_control_point_locations[b, t, 0], dest_control_point_locations[b, t, 1], :] = image[b, source_control_point_locations[b, t, 0], source_control_point_locations[b, t, 1], :]. For regularization_weight > 0, this condition is met approximately, since regularized interpolation trades off smoothness of the interpolant vs. reconstruction of the interpolant at the control points. See `tf.contrib.image.interpolate_spline` for further documentation of the interpolation_order and regularization_weight arguments. Args: image: `[batch, height, width, channels]` float `Tensor` source_control_point_locations: `[batch, num_control_points, 2]` float `Tensor` dest_control_point_locations: `[batch, num_control_points, 2]` float `Tensor` interpolation_order: polynomial order used by the spline interpolation regularization_weight: weight on smoothness regularizer in interpolation num_boundary_points: How many zero-flow boundary points to include at each image edge.Usage: num_boundary_points=0: don't add zero-flow points num_boundary_points=1: 4 corners of the image num_boundary_points=2: 4 corners and one in the middle of each edge (8 points total) num_boundary_points=n: 4 corners and n-1 along each edge name: A name for the operation (optional). Note that image and offsets can be of type tf.half, tf.float32, or tf.float64, and do not necessarily have to be the same type. Returns: warped_image: `[batch, height, width, channels]` float `Tensor` with same type as input image. flow_field: `[batch, height, width, 2]` float `Tensor` containing the dense flow field produced by the interpolation. """ image = ops.convert_to_tensor(image) source_control_point_locations = ops.convert_to_tensor( source_control_point_locations) dest_control_point_locations = ops.convert_to_tensor( dest_control_point_locations) control_point_flows = (dest_control_point_locations - source_control_point_locations) clamp_boundaries = num_boundary_points > 0 boundary_points_per_edge = num_boundary_points - 1 with ops.name_scope(name): image_shape = tf.shape(image) batch_size, image_height, image_width = image_shape[0], image_shape[ 1], image_shape[2] # This generates the dense locations where the interpolant # will be evaluated. grid_locations = _get_grid_locations(image_height, image_width) flattened_grid_locations = tf.reshape( grid_locations, [tf.multiply(image_height, image_width), 2]) # flattened_grid_locations = constant_op.constant( # _expand_to_minibatch(flattened_grid_locations, batch_size), image.dtype) flattened_grid_locations = _expand_to_minibatch( flattened_grid_locations, batch_size) flattened_grid_locations = tf.cast(flattened_grid_locations, dtype=image.dtype) if clamp_boundaries: (dest_control_point_locations, control_point_flows) = _add_zero_flow_controls_at_boundary( dest_control_point_locations, control_point_flows, image_height, image_width, boundary_points_per_edge) flattened_flows = interpolate_spline.interpolate_spline( dest_control_point_locations, control_point_flows, flattened_grid_locations, interpolation_order, regularization_weight) dense_flows = array_ops.reshape( flattened_flows, [batch_size, image_height, image_width, 2]) warped_image = dense_image_warp.dense_image_warp(image, dense_flows) return warped_image, dense_flows
def sparse_image_warp(image, source_control_point_locations, dest_control_point_locations, interpolation_order=2, regularization_weight=0.0, num_boundary_points=0, name='sparse_image_warp'): """Image warping using correspondences between sparse control points. Apply a non-linear warp to the image, where the warp is specified by the source and destination locations of a (potentially small) number of control points. First, we use a polyharmonic spline (`tf.contrib.image.interpolate_spline`) to interpolate the displacements between the corresponding control points to a dense flow field. Then, we warp the image using this dense flow field (`tf.contrib.image.dense_image_warp`). Let t index our control points. For regularization_weight=0, we have: warped_image[b, dest_control_point_locations[b, t, 0], dest_control_point_locations[b, t, 1], :] = image[b, source_control_point_locations[b, t, 0], source_control_point_locations[b, t, 1], :]. For regularization_weight > 0, this condition is met approximately, since regularized interpolation trades off smoothness of the interpolant vs. reconstruction of the interpolant at the control points. See `tf.contrib.image.interpolate_spline` for further documentation of the interpolation_order and regularization_weight arguments. Args: image: `[batch, height, width, channels]` float `Tensor` source_control_point_locations: `[batch, num_control_points, 2]` float `Tensor` dest_control_point_locations: `[batch, num_control_points, 2]` float `Tensor` interpolation_order: polynomial order used by the spline interpolation regularization_weight: weight on smoothness regularizer in interpolation num_boundary_points: How many zero-flow boundary points to include at each image edge.Usage: num_boundary_points=0: don't add zero-flow points num_boundary_points=1: 4 corners of the image num_boundary_points=2: 4 corners and one in the middle of each edge (8 points total) num_boundary_points=n: 4 corners and n-1 along each edge name: A name for the operation (optional). Note that image and offsets can be of type tf.half, tf.float32, or tf.float64, and do not necessarily have to be the same type. Returns: warped_image: `[batch, height, width, channels]` float `Tensor` with same type as input image. flow_field: `[batch, height, width, 2]` float `Tensor` containing the dense flow field produced by the interpolation. """ image = ops.convert_to_tensor(image) source_control_point_locations = ops.convert_to_tensor( source_control_point_locations) dest_control_point_locations = ops.convert_to_tensor( dest_control_point_locations) control_point_flows = ( dest_control_point_locations - source_control_point_locations) clamp_boundaries = num_boundary_points > 0 boundary_points_per_edge = num_boundary_points - 1 with ops.name_scope(name): batch_size, image_height, image_width, _ = image.get_shape().as_list() # This generates the dense locations where the interpolant # will be evaluated. grid_locations = _get_grid_locations(image_height, image_width) flattened_grid_locations = np.reshape(grid_locations, [image_height * image_width, 2]) flattened_grid_locations = constant_op.constant( _expand_to_minibatch(flattened_grid_locations, batch_size), image.dtype) if clamp_boundaries: (dest_control_point_locations, control_point_flows) = _add_zero_flow_controls_at_boundary( dest_control_point_locations, control_point_flows, image_height, image_width, boundary_points_per_edge) flattened_flows = interpolate_spline.interpolate_spline( dest_control_point_locations, control_point_flows, flattened_grid_locations, interpolation_order, regularization_weight) dense_flows = array_ops.reshape(flattened_flows, [batch_size, image_height, image_width, 2]) warped_image = dense_image_warp.dense_image_warp(image, dense_flows) return warped_image, dense_flows