def test_raisesExceptionWithIncompatibleDimensions(self):
''' correlation only accepts 4-dimension tensors, with dimensions (batch_size,height,width,num_channels) '''
with tf.Session(''):
with self.assertRaises(ValueError):
cl.corr([1, 2], [[1, 2], [3, 4]]).eval()
with self.assertRaises(ValueError):
self.assertRaises(
cl.corr([1, 2], [1, 2, 3, 4]).eval(), ValueError)
with self.assertRaises(ValueError):
self.assertRaises(
cl.corr([1, 2, 3], [[1, 2], [3, 4]]).eval(), ValueError)
def test_correlationGradientAHardCodedLarge(self):
with tf.Session('') as sess:
batch_size = 1
height = 41
width = 41
depth = 1
stride = 4
max_displacement = 24
expected_depth = (2 * int(max_displacement / stride) + 1)**2
offsets = []
for row_offset in np.arange(-max_displacement,
max_displacement + 1, stride):
for col_offset in np.arange(-max_displacement,
max_displacement + 1, stride):
offsets.append((row_offset, col_offset))
my_shape = (batch_size, height, width, depth)
a = tf.placeholder(tf.float32, shape=my_shape)
feed_a = np.ones(my_shape, dtype=np.float32)
b = 2 * np.ones(my_shape, dtype=np.float32)
result = cl.corr(a,
b,
stride=stride,
max_displacement=max_displacement)
self.assertEqual(int(result.shape[3]), expected_depth)
# Check if it's aligned at all offsets
for offset_index, offset in enumerate(offsets):
result_slices = result[:, :, :, offset_index]
grad_a = tf.gradients(result_slices, a)
gradient_a = sess.run(grad_a, feed_dict={a: feed_a})
row_offset = offset[0]
col_offset = offset[1]
a_row_begin = 0
a_row_end = height - row_offset
b_row_begin = row_offset
b_row_end = height
a_col_begin = 0
a_col_end = width - col_offset
b_col_begin = col_offset
b_col_end = width
if (row_offset < 0):
a_row_begin = -row_offset
a_row_end = height
b_row_begin = 0
b_row_end = height + row_offset
if (col_offset < 0):
a_col_begin = -col_offset
a_col_end = width
b_col_begin = 0
b_col_end = width + col_offset
final_height = a_row_end - a_row_begin
final_width = a_col_end - a_col_begin
np.testing.assert_array_equal(
gradient_a[0][0, a_row_begin:a_row_end,
a_col_begin:a_col_end, 0], 2 * np.ones(
(final_height, final_width)))
def test_correlationRandom(self):
with tf.Session(''):
batch_size = 1
height = 21
width = 21
depth = 1
my_shape = (batch_size, height, width, depth)
stride = 2
max_displacement = 20
expected_depth = (2 * int(max_displacement / stride) + 1)**2
offsets = []
for row_offset in np.arange(-max_displacement,
max_displacement + 1, stride):
for col_offset in np.arange(-max_displacement,
max_displacement + 1, stride):
offsets.append((row_offset, col_offset))
for i in range(10):
a_rand = np.random.randint(10, size=my_shape)
b_rand = np.random.randint(10, size=my_shape)
result = cl.corr(a_rand,
b_rand,
stride=stride,
max_displacement=max_displacement).eval()
self.assertEqual(result.shape[3], expected_depth)
for offset_index, offset in enumerate(offsets):
row_offset = offset[0]
col_offset = offset[1]
a_row_begin = 0
a_row_end = height - row_offset
b_row_begin = row_offset
b_row_end = height
a_col_begin = 0
a_col_end = width - col_offset
b_col_begin = col_offset
b_col_end = width
if (row_offset < 0):
a_row_begin = -row_offset
a_row_end = height
b_row_begin = 0
b_row_end = height + row_offset
if (col_offset < 0):
a_col_begin = -col_offset
a_col_end = width
b_col_begin = 0
b_col_end = width + col_offset
a_slice = a_rand[:, a_row_begin:a_row_end,
a_col_begin:a_col_end, :]
b_slice = b_rand[:, b_row_begin:b_row_end,
b_col_begin:b_col_end, :]
result_rand_full = a_slice * b_slice
result_rand = np.sum(result_rand_full, axis=-1) / depth
np.testing.assert_array_equal(
result[0, a_row_begin:a_row_end, a_col_begin:a_col_end,
offset_index], result_rand[0, :, :])
def test_correlationGradientBRandom(self):
with tf.Session('') as sess:
batch_size = 1
height = 21
width = 21
depth = 1
stride = 2
max_displacement = 20
expected_depth = (2 * int(max_displacement / stride) + 1)**2
stride = 2
max_displacement = 20
expected_depth = (2 * int(max_displacement / stride) + 1)**2
offsets = []
for row_offset in np.arange(-max_displacement,
max_displacement + 1, stride):
for col_offset in np.arange(-max_displacement,
max_displacement + 1, stride):
offsets.append((row_offset, col_offset))
my_shape = (batch_size, height, width, depth)
a = np.random.randint(10, size=my_shape).astype(np.float32)
feed_b = np.random.randint(10, size=my_shape).astype(np.float32)
b = tf.placeholder(tf.float32, shape=my_shape)
result = cl.corr(a, b)
# Check if it's aligned at all offsets
for offset_index, offset in enumerate(offsets):
row_offset = offset[0]
col_offset = offset[1]
a_row_begin = 0
a_row_end = height - row_offset
b_row_begin = row_offset
b_row_end = height
a_col_begin = 0
a_col_end = width - col_offset
b_col_begin = col_offset
b_col_end = width
if (row_offset < 0):
a_row_begin = -row_offset
a_row_end = height
b_row_begin = 0
b_row_end = height + row_offset
if (col_offset < 0):
a_col_begin = -col_offset
a_col_end = width
b_col_begin = 0
b_col_end = width + col_offset
result_slice = result[:, :, :, offset_index]
grad_b = tf.gradients(result_slice, b)
gradient_b = sess.run(grad_b, feed_dict={b: feed_b})
np.testing.assert_array_equal(
a[0, a_row_begin:a_row_end, a_col_begin:a_col_end, 0],
gradient_b[0][0, b_row_begin:b_row_end,
b_col_begin:b_col_end, 0])
def test_raisesExceptionWithTooManyOffsets(self):
''' correlation only accepts up to 2601==51x51 offsets '''
with tf.Session(''):
with self.assertRaises(ValueError):
# Current max_displacement/stride is 25
my_shape = (1, 21, 21, 1)
a = np.ones((my_shape))
b = np.ones((my_shape))
self.assertRaises(cl.corr(a, b, stride=1, max_displacement=50),
ValueError)
def test_correlationHardCoded(self):
with tf.Session(''):
batch_size = 1
width = 21
depth = 1
stride = 2
max_displacement = 20
expected_depth = (2 * int(max_displacement / stride) + 1)**2
my_shape = (batch_size, width, width, depth)
result = cl.corr(np.ones(my_shape), np.ones(my_shape)).eval()
self.assertEqual(result.shape[0], my_shape[0])
self.assertEqual(result.shape[1], my_shape[1])
self.assertEqual(result.shape[2], my_shape[2])
self.assertEqual(result.shape[3], expected_depth)
self.assertEqual(result[0, 0, 0, 220], 1)
self.assertEqual(result[0, 0, 0, 0], 0)
np.testing.assert_array_equal(result[0, :, :, 220],
np.ones((width, width)))
def getEncoderModel(height=384, width=512, batch_size=32):
print "Generating model with height={}, width={},batch_size={}".format(
height, width, batch_size)
## convolution model
conv_activation = lambda x: activations.relu(
x, alpha=0.1
) # Use the activation from the FlowNetC Caffe implementation
#conv_activation = "elu"
# left and model
input_l = Input(batch_shape=(batch_size, height, width, 3),
name='pre_input')
input_r = Input(batch_shape=(batch_size, height, width, 3),
name='nxt_input')
#layer 1, output of layer 1 is height/2 x width/2
conv1 = Convolution2D(64, (7, 7),
strides=2,
batch_size=batch_size,
padding='same',
name='conv1',
activation=conv_activation)
conv1_l = conv1(input_l)
conv1_r = conv1(input_r)
#layer 2 output of layer 2 is height/4 x width/4
conv2 = Convolution2D(128, (5, 5),
strides=2,
padding='same',
name='conv2',
activation=conv_activation)
conv2_l = conv2(conv1_l)
conv2_r = conv2(conv1_r)
#layer 3 output of layer 3 is height/8 x width8
conv3 = Convolution2D(256, (5, 5),
strides=2,
padding='same',
name='conv3',
activation=conv_activation)
conv3_l = conv3(conv2_l)
conv3_r = conv3(conv2_r)
# merge
print "Generating Correlation layer..."
if use_custom_correlation:
corr_layer = Lambda(
lambda x: cl.corr(a=x[0], b=x[1], stride=2, max_displacement=20),
name="correlation_layer")([conv3_l, conv3_r])
else:
corr_layer = get_correlation_layer(conv3_l,
conv3_r,
max_displacement=20,
stride2=2,
height_8=height / 8,
width_8=width / 8)
# merged convolution
conv3_l_redir = Convolution2D(32, (1, 1),
name="conv_redir",
activation=conv_activation)(conv3_l)
conv3_l_with_corr = concatenate([conv3_l_redir, corr_layer],
name="concatenated_correlation")
conv3_1 = Convolution2D(256, (3, 3),
padding='same',
name='conv3_1',
activation=conv_activation)(conv3_l_with_corr)
#layer 4, output of layer 4 is height/16 x width/16
conv4 = Convolution2D(512, (3, 3),
strides=2,
padding='same',
name='conv4',
activation=conv_activation)(conv3_1)
height_16 = height / 16
width_16 = width / 16
conv4_1 = Convolution2D(512, (3, 3),
padding='same',
name='conv4_1',
activation=conv_activation)(conv4)
# layer 5, now /32
conv5 = Convolution2D(512, (3, 3),
strides=2,
padding='same',
name='conv5',
activation=conv_activation)(conv4_1)
height_32 = height_16 / 2
width_32 = width_16 / 2
conv5_1 = Convolution2D(512, (3, 3),
padding='same',
name='conv5_1',
activation=conv_activation)(conv5)
# Layer 6, now /64
conv6 = Convolution2D(1024, (3, 3),
strides=2,
padding='same',
name='conv6',
activation=conv_activation)(conv5_1)
height_64 = height_32 / 2
width_64 = width_32 / 2
print "Compiling..."
optimizer = SGD(nesterov=True, lr=0.00001, momentum=0.1, decay=0.001)
model = Model(inputs=[input_l, input_r], outputs=conv6)
model.compile(optimizer=optimizer, loss='mean_squared_error')
print "Done"
return model