def bilinear_pool(self, x1, x2):
p1 = tf.matmul(x1, self.C[0])
p2 = tf.matmul(x2, self.C[1])
pc1 = tf.complex(p1, tf.zeros_like(p1))
pc2 = tf.complex(p2, tf.zeros_like(p2))
conved = tf.batch_ifft(tf.batch_fft(pc1) * tf.batch_fft(pc2))
return tf.real(conved)
def bilinear_pool(self, x1, x2):
p1 = tf.matmul(x1, self.C[0])
p2 = tf.matmul(x2, self.C[1])
pc1 = tf.complex(p1, tf.zeros_like(p1))
pc2 = tf.complex(p2, tf.zeros_like(p2))
conved = tf.batch_ifft(tf.batch_fft(pc1) * tf.batch_fft(pc2))
return tf.real(conved)
def softmax_index1d(indices, values):
# indices: bs x height x length
# values: stuff x bs x height x length
indices_shape = shape_list(indices)
softmax_indices = softmax(indices)
softmax_indices = tf.complex(softmax_indices,
tf.zeros_like(softmax_indices))
values = tf.complex(values, tf.zeros_like(values))
fft_of_answer = tf.conj(
tf.batch_fft(softmax_indices)) * tf.batch_fft(values)
return tf.real(tf.batch_ifft(fft_of_answer))
def softmax_index2d(indices, values, reduce=False):
indices_shape = shape_list(indices)
softmax_indices = tf.reshape(
tf.nn.softmax(
tf.reshape(indices, [-1, indices_shape[-1] * indices_shape[-2]])),
indices_shape)
softmax_indices = tf.complex(softmax_indices,
tf.zeros_like(softmax_indices))
values = tf.complex(values, tf.zeros_like(values))
fft_of_answer = tf.conj(
tf.batch_fft2d(softmax_indices)) * tf.batch_fft2d(values)
if reduce:
return tf.reduce_mean(tf.real(tf.batch_ifft(fft_of_answer)), -2)
else:
return tf.real(tf.batch_ifft2d(fft_of_answer))
def bilinear_pool(x1, x2, output_size):
""" Computes approximation of bilinear pooling with respect to x1, x2.
For detailed explaination, see the paper (https://arxiv.org/abs/1511.06062)
Args:
x1: A `Tensor` with shape (batch_size, x1_size).
x2: A `Tensor` with shape ((batch_size, x2_size).
output_size: Output projection size. (`int`)
Returns:
A Tensor with shape (batch_size, output_size).
"""
p1 = count_sketch(x1, output_size)
p2 = count_sketch(x2, output_size)
pc1 = tf.complex(p1, tf.zeros_like(p1))
pc2 = tf.complex(p2, tf.zeros_like(p2))
return tf.batch_ifft(tf.batch_fft(pc1) * tf.batch_fft(pc2))
def bilinear_pool(x2, x1, output_size):
""" Computes approximation of bilinear pooling with respect to x1, x2.
For detailed explaination, see the paper (https://arxiv.org/abs/1511.06062)
Args:
x1: A `Tensor` with shape (batch_size, x1_size).
x2: A `Tensor` with shape ((batch_size, x2_size).
output_size: Output projection size. (`int`)
Returns:
A Tensor with shape (batch_size, output_size).
"""
p1 = count_sketch(x1, output_size)
p2 = count_sketch(x2, output_size)
pc1 = tf.complex(p1, tf.zeros_like(p1))
pc2 = tf.complex(p2, tf.zeros_like(p2))
conved = tf.batch_ifft(tf.batch_fft(pc1) * tf.batch_fft(pc2))
return tf.real(conved)
def _ifft(bottom, sequential, compute_size):
if sequential:
return sequential_batch_ifft(bottom, compute_size)
else:
return tf.batch_ifft(bottom)
import numpy as np
from sequential_batch_fft_ops import sequential_batch_fft, sequential_batch_ifft
compute_size = 128
x = tf.placeholder(tf.complex64, [None, None])
x_128 = tf.placeholder(tf.complex128, [None, None])
# FFT
x_fft = sequential_batch_fft(x, compute_size)
x_fft_128 = sequential_batch_fft(x_128, compute_size)
x_fft_tf = tf.batch_fft(x)
# IFFT
x_ifft = sequential_batch_ifft(x, compute_size)
x_ifft_128 = sequential_batch_ifft(x_128, compute_size)
x_ifft_tf = tf.batch_ifft(x)
# Grads
gx_fft = tf.gradients(x_fft, x)[0]
gx_fft_128 = tf.gradients(x_fft_128, x_128)[0]
gx_fft_tf = tf.gradients(x_fft_tf, x)[0]
gx_ifft = tf.gradients(x_ifft, x)[0]
gx_ifft_128 = tf.gradients(x_ifft_128, x_128)[0]
gx_ifft_tf = tf.gradients(x_ifft_tf, x)[0]
def test_shape():
print("Testing shape...")
# Test Shape inference. Output shape should be
# the same as input shape
input_pl = tf.placeholder(tf.complex64, [1000, 16000])
def __call__(self, inputs, state, scope='complex_RNN', real=True):
"""
(copying their naming conventions, mkay)
"""
# TODO: set up data types at time of model selection
# (for now:) cast inputs to complex
with vs.variable_scope(scope):
if not real:
step1 = times_diag(state, self._state_size, scope='Diag/First')
step2 = tf.batch_fft(step1, name='FFT')
# step3 = reflection(step2, self._state_size, scope='Reflection/First')
step3 = step2
permutation = vs.get_variable(
"Permutation",
dtype=tf.complex64,
initializer=tf.complex(
np.random.permutation(
np.eye(self._state_size, dtype=np.float32)),
tf.constant(0.0, dtype=tf.float32)),
trainable=False)
step4 = tf.matmul(step3, permutation)
step5 = times_diag(step4,
self._state_size,
scope='Diag/Second')
step6 = tf.batch_ifft(step5, name='InverseFFT')
step7 = reflection(step6,
self._state_size,
scope='Reflection/Second')
step8 = times_diag(step7, self._state_size, scope='Diag/Third')
step1 = state
# (folding in the input data)
foldin_re = linear(inputs,
self._state_size,
bias=False,
scope='Linear/FoldIn/Real')
foldin_im = linear(inputs,
self._state_size,
bias=False,
scope='Linear/FoldIn/Imaginary')
intermediate_state = tf.complex(
foldin_re, foldin_im,
name='Linear/Intermediate/Complex') + step8
# intermediate_re = foldin_re + tf.real(step8)
# intermediate_im = foldin_im + tf.imag(step8)
# intermediate_state = tf.concat(1, [intermediate_re, intermediate_im])
# new_state_real = relu_mod(intermediate_state, self._state_size, scope='ReLU_mod', real=True)
new_state = relu_mod(intermediate_state,
self._state_size,
scope='ReLU_mod',
name='new_state') # DEFAULT
#new_state = tf.nn.relu(intermediate_state, name='new_state')
#new_state = tf.nn.tanh(intermediate_state, name='new_state')
#new_state = tf.identity(intermediate_state, name='new_state')
#new_state = tf.maximum(0.1*intermediate_state, intermediate_state, name='new_state') # ... leaky relu
# new_state = intermediate_state
# taken from uRNN
#new_state = tanh_mod(intermediate_re, intermediate_im, scope='tanh_mod', name='new_state')
#new_state = tf.nn.sigmoid(intermediate_state, name='new_state')
#new_state = relu_mod(intermediate_state, self._state_size, scope='ReLU_mod', real=True, name='new_state')
real_state = tf.concat(
1, [tf.real(new_state),
tf.imag(new_state)])
output = linear(real_state,
self._output_size,
bias=True,
scope='Linear/Output')
# output = linear(new_state_real, self._output_size, bias=True, scope='Linear/Output')
# new_state_re = tf.slice(new_state_real, [0, 0], [-1, self._state_size])
# new_state_im = tf.slice(new_state_real, [0, self._state_size], [-1, self._state_size])
# new_state = tf.complex(new_state_re, new_state_im)
else:
# state is [state_re, state_im]
step1_re, step1_im = times_diag(state,
self._state_size,
scope='Diag/First',
real=True,
split=True) # gradient is fine
step2 = tf.div(tf.batch_fft(tf.complex(step1_re, step1_im)),
np.sqrt(self._state_size / 2),
name='FFT') # gradient looks ok
step3_re, step3_im = reflection(
step2,
self._state_size / 2,
scope='Reflection/First',
real=True,
split=True
) # gradient looks ok, but is it giving the right reflection formula?
permutation = vs.get_variable("Permutation",
dtype=tf.float32,
initializer=tf.constant(
np.random.permutation(
np.eye(
self._state_size / 2,
dtype=np.float32))),
trainable=False)
step4_re = tf.matmul(step3_re, permutation)
step4_im = tf.matmul(step3_im, permutation)
# step4 = tf.concat(1, [step4_re, step4_im])
step5_re, step5_im = times_diag((step4_re, step4_im),
self._state_size,
scope='Diag/Second',
real=True,
split=True)
step6 = tf.mul(tf.batch_ifft(tf.complex(step5_re, step5_im)),
np.sqrt(self._state_size / 2),
name='IFFT')
step7 = reflection(step6,
self._state_size / 2,
scope='Reflection/Second',
real=True,
split=False)
step8 = times_diag(step7,
self._state_size,
scope='Diag/Third',
real=True,
split=False)
foldin = linear(inputs,
self._state_size,
bias=False,
scope='Linear/FoldIn/Real')
intermediate_state = step8 + foldin
new_state = relu_mod(intermediate_state,
self._state_size,
scope='ReLU_mod',
real=True,
name='new_state')
output = linear(new_state,
self._output_size,
bias=True,
scope='Linear/Output')
return output, new_state
def compact_bilinear_pooling(bottom1,
bottom2,
output_dim,
sum_pool=True,
rand_h_1=None,
rand_s_1=None,
rand_h_2=None,
rand_s_2=None,
seed_h_1=1,
seed_s_1=3,
seed_h_2=5,
seed_s_2=7):
"""
Compute compact bilinear pooling over two bottom inputs. Reference:
Yang Gao, et al. "Compact Bilinear Pooling." in Proceedings of IEEE
Conference on Computer Vision and Pattern Recognition (2016).
Akira Fukui, et al. "Multimodal Compact Bilinear Pooling for Visual Question
Answering and Visual Grounding." arXiv preprint arXiv:1606.01847 (2016).
Args:
bottom1: 1st input, 4D Tensor of shape [batch_size, height, width, input_dim1].
bottom2: 2nd input, 4D Tensor of shape [batch_size, height, width, input_dim2].
output_dim: output dimension for compact bilinear pooling.
sum_pool: (Optional) If True, sum the output along height and width
dimensions and return output shape [batch_size, output_dim].
Otherwise return [batch_size, height, width, output_dim].
Default: True.
rand_h_1: (Optional) an 1D numpy array containing indices in interval
`[0, output_dim)`. Automatically generated from `seed_h_1`
if is None.
rand_s_1: (Optional) an 1D numpy array of 1 and -1, having the same shape
as `rand_h_1`. Automatically generated from `seed_s_1` if is
None.
rand_h_2: (Optional) an 1D numpy array containing indices in interval
`[0, output_dim)`. Automatically generated from `seed_h_2`
if is None.
rand_s_2: (Optional) an 1D numpy array of 1 and -1, having the same shape
as `rand_h_2`. Automatically generated from `seed_s_2` if is
None.
Returns:
Compact bilinear pooled results of shape [batch_size, output_dim] or
[batch_size, height, width, output_dim], depending on `sum_pool`.
"""
# Static shapes are needed to construction count sketch matrix
input_dim1 = bottom1.get_shape().as_list()[-1]
input_dim2 = bottom2.get_shape().as_list()[-1]
# Step 0: Generate vectors and sketch matrix for tensor count sketch
# This is only done once during graph construction, and fixed during each
# operation
if rand_h_1 is None:
np.random.seed(seed_h_1)
rand_h_1 = np.random.randint(output_dim, size=input_dim1)
if rand_s_1 is None:
np.random.seed(seed_s_1)
rand_s_1 = 2 * np.random.randint(2, size=input_dim1) - 1
sparse_sketch_matrix1 = _generate_sketch_matrix(rand_h_1, rand_s_1,
output_dim)
if rand_h_2 is None:
np.random.seed(seed_h_2)
rand_h_2 = np.random.randint(output_dim, size=input_dim2)
if rand_s_2 is None:
np.random.seed(seed_s_2)
rand_s_2 = 2 * np.random.randint(2, size=input_dim2) - 1
sparse_sketch_matrix2 = _generate_sketch_matrix(rand_h_2, rand_s_2,
output_dim)
# Step 1: Flatten the input tensors and count sketch
bottom1_flat = tf.reshape(bottom1, [-1, input_dim1])
bottom2_flat = tf.reshape(bottom2, [-1, input_dim2])
# Essentially:
# sketch1 = bottom1 * sparse_sketch_matrix
# sketch2 = bottom2 * sparse_sketch_matrix
# But tensorflow only supports left multiplying a sparse matrix, so:
# sketch1 = (sparse_sketch_matrix.T * bottom1.T).T
# sketch2 = (sparse_sketch_matrix.T * bottom2.T).T
sketch1 = tf.transpose(
tf.sparse_tensor_dense_matmul(sparse_sketch_matrix1,
bottom1_flat,
adjoint_a=True,
adjoint_b=True))
sketch2 = tf.transpose(
tf.sparse_tensor_dense_matmul(sparse_sketch_matrix2,
bottom2_flat,
adjoint_a=True,
adjoint_b=True))
# Step 2: FFT
zeros = tf.zeros_like(sketch1)
fft1 = tf.batch_fft(tf.complex(real=sketch1, imag=zeros))
fft2 = tf.batch_fft(tf.complex(real=sketch2, imag=zeros))
# Step 3: Elementwise product
fft_product = tf.mul(fft1, fft2)
# Step 4: Inverse FFT and reshape back
# Compute output shape dynamically: [batch_size, height, width, output_dim]
cbp_flat = tf.real(tf.batch_ifft(fft_product))
output_shape = tf.add(tf.mul(tf.shape(bottom1), [1, 1, 1, 0]),
[0, 0, 0, output_dim])
cbp = tf.reshape(cbp_flat, output_shape)
# Step 5: Sum pool over spatial dimensions, if specified
if sum_pool:
cbp = tf.reduce_sum(cbp, reduction_indices=[1, 2])
return cbp