def test_visualization(self):
if not VISUALIZE:
return
cur_dir = os.path.dirname(os.path.abspath(__file__))
root_dir = os.path.join(cur_dir, '..')
flow_path = os.path.join(root_dir, 'pwcnet', 'warp', 'test_data',
'flow_ab.flo')
image_path = os.path.join(root_dir, 'pwcnet', 'warp', 'test_data',
'image_a.png')
flow_ab = [read_flow_file(flow_path)]
img_a = [read_image(image_path, as_float=True)]
t_tensor = tf.placeholder(tf.float32, None)
flow_ab_tensor = tf.placeholder(tf.float32, np.shape(flow_ab))
img_a_tensor = tf.placeholder(tf.float32, np.shape(img_a))
warp_tensor = forward_warp(img_a_tensor, t_tensor * flow_ab_tensor)
warp = self.sess.run(warp_tensor,
feed_dict={
flow_ab_tensor: flow_ab,
img_a_tensor: img_a,
t_tensor: 1.0
})
warp = np.clip(warp[0], 0.0, 1.0)
show_image(warp)
# For writing to video.
if WRITE_TO_VIDEO:
import cv2
import mpimg
height = img_a[0].shape[0]
width = img_a[0].shape[1]
writer = cv2.VideoWriter(cur_dir + '/outputs/warped.avi',
cv2.VideoWriter_fourcc(*'MJPG'), 20,
(width, height))
steps = 60
for i in range(steps):
print('Writing video at step %d' % i)
t = i * (1.0 / float(steps))
warped = self.sess.run(warp_tensor,
feed_dict={
flow_ab_tensor: flow_ab,
img_a_tensor: img_a,
t_tensor: t
})
warped = warped[0]
warped = np.clip(warped, 0.0, 1.0)
output_path = cur_dir + '/outputs/out-%.2f.png' % t
mpimg.imsave(output_path, warped)
writer.write(cv2.imread(output_path))
writer.release()
def test_val_data_read_write(self):
data_set = InterpDataSet(self.tf_record_directory, [[1]], batch_size=2)
preprocessor = DavisDataSetPreprocessor(self.tf_record_directory,
[[1], [1, 0], [1, 0, 0]],
shard_size=5,
validation_size=2)
preprocessor.preprocess_raw(self.data_directory)
output_paths = data_set.get_tf_record_names()
[
self.assertTrue(os.path.isfile(output_path))
for output_path in output_paths
]
# We're using a larger shard size, so there should be 1 path for val and 1 for train.
self.assertEqual(len(output_paths), 2)
data_set.load(self.sess)
data_set.init_validation_data(self.sess)
next_sequence_tensor, next_sequence_timing_tensor = data_set.get_next_batch(
)
query = [next_sequence_tensor, next_sequence_timing_tensor]
next_sequence, next_sequence_timing = self.sess.run(
query, feed_dict=data_set.get_validation_feed_dict())
self.assertListEqual(next_sequence_timing[0].tolist(), [0.0, 0.5, 1.0])
self.assertListEqual(next_sequence_timing[1].tolist(), [0.0, 0.5, 1.0])
self.assertTupleEqual(np.shape(next_sequence), (2, 3, 256, 256, 3))
if VISUALIZE:
print(
'Showing dense sequence (val set) with timings [0.0, 0.5, 1.0] ...'
)
for j in range(3):
show_image(next_sequence[0][j])
end_of_val = False
try:
next_sequence, next_sequence_timing = self.sess.run(
query, feed_dict=data_set.get_validation_feed_dict())
except tf.errors.OutOfRangeError:
end_of_val = True
self.assertTrue(end_of_val)
def single_warp_test_helper(self, flow_ab_path, img_a_path, img_b_path,
tolerance):
"""
Runs warp test for a set of 2 images and a flow between them.
Also masks out the occluded warp regions when computing the error.
:param flow_ab_path: Str.
:param img_a_path: Str.
:param img_b_path: Str.
:param tolerance: Float. Usually a percentage like 0.03.
"""
# Load from files.
flow_ab = read_flow_file(flow_ab_path)
img_a = read_image(img_a_path, as_float=True)
img_b = read_image(img_b_path, as_float=True)
mask_ab = np.ones(shape=img_a.shape)
H = img_a.shape[0]
W = img_a.shape[1]
C = img_a.shape[2]
# Create the graph.
img_shape = [None, H, W, C]
flow_shape = [None, H, W, 2]
input = tf.placeholder(shape=img_shape, dtype=tf.float32)
flow_tensor = tf.placeholder(shape=flow_shape, dtype=tf.float32)
warped_tensor = backward_warp(input, flow_tensor)
# Run.
warped_image = self.sess.run(warped_tensor,
feed_dict={
input: [img_b, mask_ab],
flow_tensor: [flow_ab, flow_ab]
})
# Get the masked errors.
error_ab_img = np.abs(warped_image[0] - img_a) * warped_image[1]
error_ab = np.mean(error_ab_img)
# Assert a < tolerance average error.
self.assertLess(error_ab, tolerance)
if SHOW_WARPED_IMAGES:
show_image(warped_image[0])
show_image(error_ab_img)
def test_pyramid(self):
num_levels = 5
pyr_builder = LaplacianPyramid(num_levels, filter_side_len=5)
image_height = self.test_image.shape[0]
image_width = self.test_image.shape[1]
image_channels = self.test_image.shape[2]
image_tensor = tf.placeholder(tf.float32,
shape=(1, image_height, image_width,
image_channels))
pyr_tensors, _, reconstructed_tensor = pyr_builder.get_forward(
image_tensor)
pyr = self.sess.run(pyr_tensors,
feed_dict={image_tensor: [self.test_image]})
reconstructed = self.sess.run(
reconstructed_tensor, feed_dict={image_tensor: [self.test_image]})
# Check shapes.
self.assertEqual(len(pyr), num_levels)
for i, level in enumerate(pyr):
self.assertTupleEqual(
np.shape(level),
(1, image_height / 2**i, image_width / 2**i, image_channels))
# Check that gradients flow.
grads_tensor = tf.gradients(pyr_tensors, image_tensor)
grads = self.sess.run(grads_tensor,
feed_dict={image_tensor: [self.test_image]})
for grad in grads:
self.assertNotEqual(np.sum(grad), 0.0)
# Check reconstruction error.
diff = np.abs(reconstructed[0] - self.test_image)
self.assertLessEqual(np.sum(diff), 2E-2)
if VISUALIZE:
print('Showing reconstruction...')
show_image(np.clip(reconstructed[0] / 255.0, 0, 255))
for i, level in enumerate(pyr):
print('Showing level %d...' % i)
show_image(np.clip(level[0] / 255.0, 0, 255))
def test_optical_flow_warp_sintel(self):
"""
Runs warp test against Sintel ground truth and checks for <2% average pixel error.
Also masks out the occluded warp regions when computing the error.
"""
# Load from files.
flow_ab = read_flow_file('pwcnet/warp/test_data/flow_ab.flo')
img_a = read_image('pwcnet/warp/test_data/image_a.png', as_float=True)
img_b = read_image('pwcnet/warp/test_data/image_b.png', as_float=True)
flow_cd = read_flow_file('pwcnet/warp/test_data/flow_cd.flo')
img_c = read_image('pwcnet/warp/test_data/image_c.png', as_float=True)
img_d = read_image('pwcnet/warp/test_data/image_d.png', as_float=True)
mask_ab = np.ones(shape=img_a.shape)
mask_cd = np.ones(shape=img_a.shape)
H = img_a.shape[0]
W = img_a.shape[1]
C = img_a.shape[2]
# Create the graph.
img_shape = [None, H, W, C]
flow_shape = [None, H, W, 2]
input = tf.placeholder(shape=img_shape, dtype=tf.float32)
flow_tensor = tf.placeholder(shape=flow_shape, dtype=tf.float32)
warped_tensor = backward_warp(input, flow_tensor)
# Run.
warped_image = self.sess.run(warped_tensor,
feed_dict={
input:
[img_b, img_d, mask_ab, mask_cd],
flow_tensor:
[flow_ab, flow_cd, flow_ab, flow_cd]
})
# Get the masked errors.
error_ab_img = np.abs(warped_image[0] - img_a) * warped_image[2]
error_cd_img = np.abs(warped_image[1] - img_c) * warped_image[3]
error_ab = np.mean(error_ab_img)
error_cd = np.mean(error_cd_img)
# Assert a < 1.3% average error.
self.assertLess(error_ab, 0.013)
self.assertLess(error_cd, 0.013)
if SHOW_WARPED_IMAGES:
show_image(warped_image[0])
show_image(warped_image[1])
show_image(error_ab_img)
show_image(error_cd_img)
def run_case(self, flow_ab, img_a, img_b, flow_cd, img_c, img_d):
"""
Helper test method.
"""
mask_ab = np.ones(shape=img_a.shape)
mask_cd = np.ones(shape=img_a.shape)
H = img_a.shape[0]
W = img_a.shape[1]
C = img_a.shape[2]
# Create the graph.
img_shape = [None, H, W, C]
flow_shape = [None, H, W, 2]
input = tf.placeholder(shape=img_shape, dtype=tf.float32)
flow_tensor = tf.placeholder(shape=flow_shape, dtype=tf.float32)
warped_tensor = backward_warp(input, flow_tensor)
# Run.
feed_dict = {
input: [img_b, img_d, mask_ab, mask_cd],
flow_tensor: [flow_ab, flow_cd, flow_ab, flow_cd]
}
warped_image = self.sess.run(warped_tensor, feed_dict=feed_dict)
# Get the masked errors.
error_ab_img = np.abs(warped_image[0] - img_a) * warped_image[2]
error_cd_img = np.abs(warped_image[1] - img_c) * warped_image[3]
error_ab = np.mean(error_ab_img)
error_cd = np.mean(error_cd_img)
# Assert a < 1.3% average error.
self.assertLess(error_ab, 0.013)
self.assertLess(error_cd, 0.013)
if SHOW_AUGMENTATION_DEBUG_IMAGES:
show_image(warped_image[0])
show_image(warped_image[1])
show_image(error_ab_img)
show_image(error_cd_img)
def test_single_transform(self):
"""
This test creates a box and translates it in 2 ways:
1) Regular translation via numpy.
2) Translation via SpacialTransformNetwork.
Then, it diffs both translation methods and asserts that they are close.
"""
# Image dimensions.
height = 70
width = 90
channels = 3
image_shape = [height, width, channels]
# Defines how many pixels to translate by.
x_trans_amount = 20
y_trans_amount = 10
# Defines the square position and size.
y_start = 10
y_end = 40
x_start = 20
x_end = 50
# The maximum diff is the circumference of the square.
max_diff = (x_end - x_start) * 2 + (y_end - y_start) * 2
# Input.
box_image = np.zeros(shape=image_shape, dtype=np.float)
box_image[y_start:y_end, x_start:x_end, :] = 1.0
# Expected output.
translated_box_image = np.zeros(shape=image_shape, dtype=np.float)
translated_box_image[y_start + y_trans_amount:y_end + y_trans_amount,
x_start + x_trans_amount:x_end +
x_trans_amount, :] = 1.0
# Warp matrix to achieve the expected output.
translation = np.asarray([[[[x_trans_amount, y_trans_amount]]]])
translation = np.tile(translation, [1, height, width, 1])
warp = translation[0, 0, 0, :]
# Create the graph and run it to get the actual output.
input = tf.placeholder(shape=[None] + image_shape, dtype=tf.float32)
theta = tf.placeholder(shape=[None, 2], dtype=tf.float32)
transformed = spatial_transformer_network(input, theta)
# Run with batch size of 2.
transformed_image = self.sess.run(
transformed,
feed_dict={
input: [translated_box_image, translated_box_image],
theta: [warp, warp]
})
# Do the diff.
diff_img = np.abs(transformed_image[0] - box_image)
diff = np.sum(np.mean(diff_img, axis=-1))
self.assertLess(diff, max_diff)
diff_img = np.abs(transformed_image[1] - box_image)
diff = np.sum(np.mean(diff_img, axis=-1))
self.assertLess(diff, max_diff)
if SHOW_WARPED_IMAGES:
show_image(transformed_image[1])
show_image(diff_img)