def testEvolvedTransformerEncoderLayerFProp(self):
with self.session(use_gpu=True) as sess:
np.random.seed(6348575)
depth = 4
p = GPipeEvolvedTransformerEncoderLayer.Params()
p.name = 'gpipe_evolved_transformer_encoder'
p.source_dim = depth
p.transformer_tpl.tr_fflayer_tpl.hidden_dim = 7
p.transformer_tpl.tr_atten_tpl.num_attention_heads = 2
transformer = GPipeEvolvedTransformerEncoderLayer(p)
(source_vecs, source_padding, _, _) = self._testInputs(depth=depth)
h = transformer.FPropDefaultTheta(source_vecs, source_padding, None)[0]
tf.global_variables_initializer().run()
actual_layer_output = sess.run([h])[0]
tf.logging.info(np.array_repr(actual_layer_output))
# pylint: disable=bad-whitespace
# pyformat: disable
expected_layer_output = [
[[-2.03854632, -1.07184005, -0.28417355, 0.17936069],
[-0.74067241, -1.48318326, 0.26369774, 0.62173623]],
[[-2.12831736, -0.86353737, -0.54453588, 0.13070297],
[-0.76326936, -0.04828247, -0.49510449, 1.20852029]],
[[ 0.85539216, -1.21577334, -1.28910851, -0.15619087],
[-1.45574117, -1.11208296, 0.71455258, 0.91494167]],
[[-1.21304905, -1.37239563, 0.7022025 , 0.16537377],
[ 3.07106829, 1.35782909, -0.9944036 , -2.28987551]],
[[-0.13129801, -1.70681071, -0.42324018, 1.32114363],
[-1.53065133, 0.18422687, -0.93387115, 1.37142754]]]
# pyformat: enable
# pylint: enable=bad-whitespace
self.assertAllClose(expected_layer_output, actual_layer_output)
def testEvolvedTransformerEncoderLayerConstruction(self):
p = GPipeEvolvedTransformerEncoderLayer.Params()
p.name = 'gpipe_evolved_transformer_encoder'
p.source_dim = 4
p.transformer_tpl.tr_fflayer_tpl.hidden_dim = 7
p.transformer_tpl.tr_atten_tpl.num_attention_heads = 2
_ = GPipeEvolvedTransformerEncoderLayer(p)
def testEvolvedTransformerEncoderLayerFProp(self):
with self.session(use_gpu=True):
np.random.seed(6348575)
depth = 4
p = GPipeEvolvedTransformerEncoderLayer.Params()
p.name = 'gpipe_evolved_transformer_encoder'
p.source_dim = depth
p.transformer_tpl.tr_fflayer_tpl.hidden_dim = 7
p.transformer_tpl.tr_atten_tpl.num_attention_heads = 2
transformer = GPipeEvolvedTransformerEncoderLayer(p)
(source_vecs, source_padding, _, _, _,
_) = self._testInputs(depth=depth)
output = transformer.FPropDefaultTheta(source_vecs, source_padding,
None, None, None, None,
None, None, None, None)
h = output[0]
self.evaluate(tf.global_variables_initializer())
actual_layer_output = self.evaluate([h])[0]
tf.logging.info(np.array_repr(actual_layer_output))
# pylint: disable=bad-whitespace
# pyformat: disable
expected_layer_output = [
[[-2.059799, -1.2909635, 0.25885093, 0.14688382],
[-0.8974035, -1.5618052, 0.4086342, 0.8884169]],
[[-1.8822882, -0.0494532, -0.54840994, -0.27622807],
[-0.9241364, -0.16045655, -0.32357514, 1.485477]],
[[0.8964625, -1.1524196, -1.0661578, -0.22737658],
[-1.6575948, -1.1755587, 0.93549323, 0.8066237]],
[[-1.1761202, -1.3182621, 0.79037327, -0.03893514],
[2.1135292, 1.3683249, -0.24891634, -2.525507]],
[[0.24391544, -1.6017915, -0.7599152, 1.1556216],
[-1.6751958, 0.04410481, -0.8073819, 1.6209301]]
]
# pyformat: enable
# pylint: enable=bad-whitespace
self.assertAllClose(expected_layer_output,
actual_layer_output,
rtol=1e-05,
atol=1e-05)
def testEvolvedTransformerEncoderLayerFProp(self):
with self.session(use_gpu=True) as sess:
np.random.seed(6348575)
depth = 4
p = GPipeEvolvedTransformerEncoderLayer.Params()
p.name = 'gpipe_evolved_transformer_encoder'
p.source_dim = depth
p.transformer_tpl.tr_fflayer_tpl.hidden_dim = 7
p.transformer_tpl.tr_atten_tpl.num_attention_heads = 2
transformer = GPipeEvolvedTransformerEncoderLayer(p)
(source_vecs, source_padding, _, _, _,
_) = self._testInputs(depth=depth)
output = transformer.FPropDefaultTheta(source_vecs, source_padding,
None, None, None, None,
None, None, None, None)
h = output[0]
tf.global_variables_initializer().run()
actual_layer_output = sess.run([h])[0]
tf.logging.info(np.array_repr(actual_layer_output))
# pylint: disable=bad-whitespace
# pyformat: disable
expected_layer_output = [
[[-2.05546284, -1.15067506, -0.00898855, 0.26481438],
[-0.8181392, -1.40835416, 0.47274107, 0.86176264]],
[[-1.43251371, 0.69686228, -0.70026731, -0.47239268],
[-0.39946821, -0.27037358, -0.22701442, 1.33816898]],
[[0.89412129, -1.07294774, -0.86541933, -0.21121001],
[-1.55683649, -1.14919782, 0.95287859, 0.11334917]],
[[-1.11910486, -1.01226425, 0.68622279, 0.00536875],
[2.33264184, 1.45991778, -0.71802276, -1.77120328]],
[[0.52163047, -1.90511549, -0.56069887, 1.1521647],
[-1.49890876, 0.20399603, -0.78263998, 1.30187178]]
]
# pyformat: enable
# pylint: enable=bad-whitespace
self.assertAllClose(expected_layer_output, actual_layer_output)