def test_visualization(self): if not VISUALIZE: return cur_dir = os.path.dirname(os.path.abspath(__file__)) root_dir = os.path.join(cur_dir, '..') flow_path = os.path.join(root_dir, 'pwcnet', 'warp', 'test_data', 'flow_ab.flo') image_path = os.path.join(root_dir, 'pwcnet', 'warp', 'test_data', 'image_a.png') flow_ab = [read_flow_file(flow_path)] img_a = [read_image(image_path, as_float=True)] t_tensor = tf.placeholder(tf.float32, None) flow_ab_tensor = tf.placeholder(tf.float32, np.shape(flow_ab)) img_a_tensor = tf.placeholder(tf.float32, np.shape(img_a)) warp_tensor = forward_warp(img_a_tensor, t_tensor * flow_ab_tensor) warp = self.sess.run(warp_tensor, feed_dict={ flow_ab_tensor: flow_ab, img_a_tensor: img_a, t_tensor: 1.0 }) warp = np.clip(warp[0], 0.0, 1.0) show_image(warp) # For writing to video. if WRITE_TO_VIDEO: import cv2 import mpimg height = img_a[0].shape[0] width = img_a[0].shape[1] writer = cv2.VideoWriter(cur_dir + '/outputs/warped.avi', cv2.VideoWriter_fourcc(*'MJPG'), 20, (width, height)) steps = 60 for i in range(steps): print('Writing video at step %d' % i) t = i * (1.0 / float(steps)) warped = self.sess.run(warp_tensor, feed_dict={ flow_ab_tensor: flow_ab, img_a_tensor: img_a, t_tensor: t }) warped = warped[0] warped = np.clip(warped, 0.0, 1.0) output_path = cur_dir + '/outputs/out-%.2f.png' % t mpimg.imsave(output_path, warped) writer.write(cv2.imread(output_path)) writer.release()
def test_val_data_read_write(self): data_set = InterpDataSet(self.tf_record_directory, [[1]], batch_size=2) preprocessor = DavisDataSetPreprocessor(self.tf_record_directory, [[1], [1, 0], [1, 0, 0]], shard_size=5, validation_size=2) preprocessor.preprocess_raw(self.data_directory) output_paths = data_set.get_tf_record_names() [ self.assertTrue(os.path.isfile(output_path)) for output_path in output_paths ] # We're using a larger shard size, so there should be 1 path for val and 1 for train. self.assertEqual(len(output_paths), 2) data_set.load(self.sess) data_set.init_validation_data(self.sess) next_sequence_tensor, next_sequence_timing_tensor = data_set.get_next_batch( ) query = [next_sequence_tensor, next_sequence_timing_tensor] next_sequence, next_sequence_timing = self.sess.run( query, feed_dict=data_set.get_validation_feed_dict()) self.assertListEqual(next_sequence_timing[0].tolist(), [0.0, 0.5, 1.0]) self.assertListEqual(next_sequence_timing[1].tolist(), [0.0, 0.5, 1.0]) self.assertTupleEqual(np.shape(next_sequence), (2, 3, 256, 256, 3)) if VISUALIZE: print( 'Showing dense sequence (val set) with timings [0.0, 0.5, 1.0] ...' ) for j in range(3): show_image(next_sequence[0][j]) end_of_val = False try: next_sequence, next_sequence_timing = self.sess.run( query, feed_dict=data_set.get_validation_feed_dict()) except tf.errors.OutOfRangeError: end_of_val = True self.assertTrue(end_of_val)
def single_warp_test_helper(self, flow_ab_path, img_a_path, img_b_path, tolerance): """ Runs warp test for a set of 2 images and a flow between them. Also masks out the occluded warp regions when computing the error. :param flow_ab_path: Str. :param img_a_path: Str. :param img_b_path: Str. :param tolerance: Float. Usually a percentage like 0.03. """ # Load from files. flow_ab = read_flow_file(flow_ab_path) img_a = read_image(img_a_path, as_float=True) img_b = read_image(img_b_path, as_float=True) mask_ab = np.ones(shape=img_a.shape) H = img_a.shape[0] W = img_a.shape[1] C = img_a.shape[2] # Create the graph. img_shape = [None, H, W, C] flow_shape = [None, H, W, 2] input = tf.placeholder(shape=img_shape, dtype=tf.float32) flow_tensor = tf.placeholder(shape=flow_shape, dtype=tf.float32) warped_tensor = backward_warp(input, flow_tensor) # Run. warped_image = self.sess.run(warped_tensor, feed_dict={ input: [img_b, mask_ab], flow_tensor: [flow_ab, flow_ab] }) # Get the masked errors. error_ab_img = np.abs(warped_image[0] - img_a) * warped_image[1] error_ab = np.mean(error_ab_img) # Assert a < tolerance average error. self.assertLess(error_ab, tolerance) if SHOW_WARPED_IMAGES: show_image(warped_image[0]) show_image(error_ab_img)
def test_pyramid(self): num_levels = 5 pyr_builder = LaplacianPyramid(num_levels, filter_side_len=5) image_height = self.test_image.shape[0] image_width = self.test_image.shape[1] image_channels = self.test_image.shape[2] image_tensor = tf.placeholder(tf.float32, shape=(1, image_height, image_width, image_channels)) pyr_tensors, _, reconstructed_tensor = pyr_builder.get_forward( image_tensor) pyr = self.sess.run(pyr_tensors, feed_dict={image_tensor: [self.test_image]}) reconstructed = self.sess.run( reconstructed_tensor, feed_dict={image_tensor: [self.test_image]}) # Check shapes. self.assertEqual(len(pyr), num_levels) for i, level in enumerate(pyr): self.assertTupleEqual( np.shape(level), (1, image_height / 2**i, image_width / 2**i, image_channels)) # Check that gradients flow. grads_tensor = tf.gradients(pyr_tensors, image_tensor) grads = self.sess.run(grads_tensor, feed_dict={image_tensor: [self.test_image]}) for grad in grads: self.assertNotEqual(np.sum(grad), 0.0) # Check reconstruction error. diff = np.abs(reconstructed[0] - self.test_image) self.assertLessEqual(np.sum(diff), 2E-2) if VISUALIZE: print('Showing reconstruction...') show_image(np.clip(reconstructed[0] / 255.0, 0, 255)) for i, level in enumerate(pyr): print('Showing level %d...' % i) show_image(np.clip(level[0] / 255.0, 0, 255))
def test_optical_flow_warp_sintel(self): """ Runs warp test against Sintel ground truth and checks for <2% average pixel error. Also masks out the occluded warp regions when computing the error. """ # Load from files. flow_ab = read_flow_file('pwcnet/warp/test_data/flow_ab.flo') img_a = read_image('pwcnet/warp/test_data/image_a.png', as_float=True) img_b = read_image('pwcnet/warp/test_data/image_b.png', as_float=True) flow_cd = read_flow_file('pwcnet/warp/test_data/flow_cd.flo') img_c = read_image('pwcnet/warp/test_data/image_c.png', as_float=True) img_d = read_image('pwcnet/warp/test_data/image_d.png', as_float=True) mask_ab = np.ones(shape=img_a.shape) mask_cd = np.ones(shape=img_a.shape) H = img_a.shape[0] W = img_a.shape[1] C = img_a.shape[2] # Create the graph. img_shape = [None, H, W, C] flow_shape = [None, H, W, 2] input = tf.placeholder(shape=img_shape, dtype=tf.float32) flow_tensor = tf.placeholder(shape=flow_shape, dtype=tf.float32) warped_tensor = backward_warp(input, flow_tensor) # Run. warped_image = self.sess.run(warped_tensor, feed_dict={ input: [img_b, img_d, mask_ab, mask_cd], flow_tensor: [flow_ab, flow_cd, flow_ab, flow_cd] }) # Get the masked errors. error_ab_img = np.abs(warped_image[0] - img_a) * warped_image[2] error_cd_img = np.abs(warped_image[1] - img_c) * warped_image[3] error_ab = np.mean(error_ab_img) error_cd = np.mean(error_cd_img) # Assert a < 1.3% average error. self.assertLess(error_ab, 0.013) self.assertLess(error_cd, 0.013) if SHOW_WARPED_IMAGES: show_image(warped_image[0]) show_image(warped_image[1]) show_image(error_ab_img) show_image(error_cd_img)
def run_case(self, flow_ab, img_a, img_b, flow_cd, img_c, img_d): """ Helper test method. """ mask_ab = np.ones(shape=img_a.shape) mask_cd = np.ones(shape=img_a.shape) H = img_a.shape[0] W = img_a.shape[1] C = img_a.shape[2] # Create the graph. img_shape = [None, H, W, C] flow_shape = [None, H, W, 2] input = tf.placeholder(shape=img_shape, dtype=tf.float32) flow_tensor = tf.placeholder(shape=flow_shape, dtype=tf.float32) warped_tensor = backward_warp(input, flow_tensor) # Run. feed_dict = { input: [img_b, img_d, mask_ab, mask_cd], flow_tensor: [flow_ab, flow_cd, flow_ab, flow_cd] } warped_image = self.sess.run(warped_tensor, feed_dict=feed_dict) # Get the masked errors. error_ab_img = np.abs(warped_image[0] - img_a) * warped_image[2] error_cd_img = np.abs(warped_image[1] - img_c) * warped_image[3] error_ab = np.mean(error_ab_img) error_cd = np.mean(error_cd_img) # Assert a < 1.3% average error. self.assertLess(error_ab, 0.013) self.assertLess(error_cd, 0.013) if SHOW_AUGMENTATION_DEBUG_IMAGES: show_image(warped_image[0]) show_image(warped_image[1]) show_image(error_ab_img) show_image(error_cd_img)
def test_single_transform(self): """ This test creates a box and translates it in 2 ways: 1) Regular translation via numpy. 2) Translation via SpacialTransformNetwork. Then, it diffs both translation methods and asserts that they are close. """ # Image dimensions. height = 70 width = 90 channels = 3 image_shape = [height, width, channels] # Defines how many pixels to translate by. x_trans_amount = 20 y_trans_amount = 10 # Defines the square position and size. y_start = 10 y_end = 40 x_start = 20 x_end = 50 # The maximum diff is the circumference of the square. max_diff = (x_end - x_start) * 2 + (y_end - y_start) * 2 # Input. box_image = np.zeros(shape=image_shape, dtype=np.float) box_image[y_start:y_end, x_start:x_end, :] = 1.0 # Expected output. translated_box_image = np.zeros(shape=image_shape, dtype=np.float) translated_box_image[y_start + y_trans_amount:y_end + y_trans_amount, x_start + x_trans_amount:x_end + x_trans_amount, :] = 1.0 # Warp matrix to achieve the expected output. translation = np.asarray([[[[x_trans_amount, y_trans_amount]]]]) translation = np.tile(translation, [1, height, width, 1]) warp = translation[0, 0, 0, :] # Create the graph and run it to get the actual output. input = tf.placeholder(shape=[None] + image_shape, dtype=tf.float32) theta = tf.placeholder(shape=[None, 2], dtype=tf.float32) transformed = spatial_transformer_network(input, theta) # Run with batch size of 2. transformed_image = self.sess.run( transformed, feed_dict={ input: [translated_box_image, translated_box_image], theta: [warp, warp] }) # Do the diff. diff_img = np.abs(transformed_image[0] - box_image) diff = np.sum(np.mean(diff_img, axis=-1)) self.assertLess(diff, max_diff) diff_img = np.abs(transformed_image[1] - box_image) diff = np.sum(np.mean(diff_img, axis=-1)) self.assertLess(diff, max_diff) if SHOW_WARPED_IMAGES: show_image(transformed_image[1]) show_image(diff_img)