def testTransformerFeedForwardLayer(self):
with self.session(use_gpu=True) as sess:
tf.set_random_seed(3980847392)
inputs = tf.random_normal([5, 2, 3], seed=948387483)
paddings = tf.zeros([5, 2])
p = layers_with_attention.TransformerFeedForwardLayer.Params()
p.name = 'transformer_fflayer'
p.input_dim = 3
p.hidden_dim = 7
transformer_fflayer = layers_with_attention.TransformerFeedForwardLayer(
p)
h = transformer_fflayer.FPropDefaultTheta(inputs, paddings)
tf.global_variables_initializer().run()
actual_layer_output = sess.run(h)
# pylint: disable=bad-whitespace
# pyformat: disable
expected_output = [[[-0.88366592, -0.05049637, 0.01003706],
[-0.10550675, 1.68050027, 2.29110384]],
[[-1.30083609, -0.40521634, 0.1911681],
[1.2597878, 1.45850968, 1.58734488]],
[[0.10373873, -0.2716777, 0.2314173],
[0.46293864, -0.06359965, 1.20189023]],
[[0.3673597, -0.1691664, 0.78656065],
[-1.51081395, -0.70281881, -0.9093715]],
[[-1.04800868, -0.70610946, -0.35321558],
[-1.92480004, 0.08361804, 0.62713993]]]
# pyformat: enable
# pylint: enable=bad-whitespace
print(np.array_repr(actual_layer_output))
self.assertAllClose(actual_layer_output, expected_output)
def testTransformerFeedForwardLayerSpecOutDim(self):
with self.session(use_gpu=True) as sess:
tf.set_random_seed(3980847392)
inputs = tf.random_normal([5, 2, 3], seed=948387483)
paddings = tf.zeros([5, 2])
p = layers_with_attention.TransformerFeedForwardLayer.Params()
p.name = 'transformer_fflayer'
p.input_dim = 3
p.output_dim = 5
p.hidden_dim = 7
transformer_fflayer = layers_with_attention.TransformerFeedForwardLayer(p)
h = transformer_fflayer.FPropDefaultTheta(inputs, paddings)
tf.global_variables_initializer().run()
actual_layer_output = sess.run(h)
# pylint: disable=bad-whitespace
# pyformat: disable
expected_output = [
[[ 1.42697251, 0.79269135, -0.85500956, -0.8122285 , -1.56555367],
[-1.7876718 , 0.26025945, -3.18244219, 1.34756351, 0.25739765]],
[[ 1.27962363, 0.88677615, -1.23556185, -1.06855559, -1.27293301],
[ 0.89336467, 2.46229172, 0.11302143, 1.19385004, -2.37805009]],
[[ 2.80146003, -0.66912627, 1.50160134, -2.30645609, -1.18872762],
[ 1.61967182, -0.51639485, 0.24441491, -1.0871532 , -0.95539457]],
[[ 2.03333473, -0.78205228, 0.71245927, -1.63276744, -0.91654319],
[ 1.54542768, -0.30343491, 0.10666496, -1.67965126, -0.15671858]],
[[ 1.60873222, -1.88402128, 0.79040933, -1.97199082, 0.4778356 ],
[-0.13516766, -0.42583361, -1.86275542, -1.09650302, 0.83263111]]]
# pyformat: enable
# pylint: enable=bad-whitespace
print(np.array_repr(actual_layer_output))
self.assertAllClose(actual_layer_output, expected_output)
