def test_transformer_encoder_forward(self): batch_size = 2 time_dim = 4 torch.manual_seed(self.seed) encoder = TransformerEncoder( hidden_size=self.hidden_size, ff_size=self.ff_size, num_layers=self.num_layers, num_heads=self.num_heads, dropout=self.dropout, emb_dropout=self.dropout) for p in encoder.parameters(): torch.nn.init.uniform_(p, -0.5, 0.5) x = torch.rand(size=(batch_size, time_dim, self.emb_size)) # no padding, no mask x_length = torch.Tensor([time_dim] * batch_size).int() mask = torch.ones([batch_size, time_dim, 1]) == 1 output, hidden = encoder(x, x_length, mask) self.assertEqual(output.shape, torch.Size( [batch_size, time_dim, self.hidden_size])) self.assertEqual(hidden, None) output_target = torch.Tensor( [[[0.1615, -0.1195, 0.0586, -0.0921, -0.3483, -0.3654, -0.6052, -0.3355, 0.3179, 0.2757, -0.2909, -0.0346], [0.1272, -0.1241, 0.0223, -0.1463, -0.3462, -0.1579, -0.5591, -0.6274, 0.1822, 0.3043, -0.3818, 0.0094], [0.0616, -0.1344, 0.0625, 0.0056, -0.2785, -0.4290, -0.5765, -0.5176, -0.0598, 0.3389, -0.5522, -0.1692], [0.1539, -0.1371, 0.0026, -0.0248, -0.0856, -0.3223, -0.5537, -0.3948, -0.2586, 0.2458, -0.2887, -0.0698]], [[0.1863, -0.1198, 0.1006, -0.0277, -0.3779, -0.3728, -0.6343, -0.3449, 0.2131, 0.2448, -0.3122, -0.1777], [0.0254, -0.1219, 0.0436, -0.0289, -0.2932, -0.2377, -0.6003, -0.5406, 0.2308, 0.3578, -0.3728, 0.0707], [0.1146, -0.1270, 0.1163, -0.0290, -0.3773, -0.3924, -0.5738, -0.6528, 0.1428, 0.3623, -0.4796, 0.0471], [0.0815, -0.1355, 0.1016, 0.0496, -0.3001, -0.4812, -0.5557, -0.6937, 0.1002, 0.2873, -0.4675, -0.1383]]] ) self.assertTensorAlmostEqual(output_target, output)
def test_transformer_encoder_forward(self): batch_size = 2 time_dim = 4 torch.manual_seed(self.seed) encoder = TransformerEncoder(hidden_size=self.hidden_size, ff_size=self.ff_size, num_layers=self.num_layers, num_heads=self.num_heads, dropout=self.dropout, emb_dropout=self.dropout) for p in encoder.parameters(): torch.nn.init.uniform_(p, -0.5, 0.5) x = torch.rand(size=(batch_size, time_dim, self.emb_size)) # no padding, no mask x_length = torch.Tensor([time_dim] * batch_size).int() mask = torch.ones([batch_size, time_dim, 1]) == 1 output, hidden = encoder(x, x_length, mask) self.assertEqual(output.shape, torch.Size([batch_size, time_dim, self.hidden_size])) self.assertEqual(hidden, None) output_target = torch.Tensor( [[[ 1.9728e-01, -1.2042e-01, 8.0998e-02, 1.3411e-03, -3.5960e-01, -5.2988e-01, -5.6056e-01, -3.5297e-01, 2.6680e-01, 2.8343e-01, -3.7342e-01, -5.9112e-03 ], [ 8.9687e-02, -1.2491e-01, 7.7809e-02, -1.3500e-03, -2.7002e-01, -4.7312e-01, -5.7981e-01, -4.1998e-01, 1.0457e-01, 2.9726e-01, -3.9461e-01, 8.1598e-02 ], [ 3.4988e-02, -1.3020e-01, 6.0043e-02, 2.7782e-02, -3.1483e-01, -3.8940e-01, -5.5557e-01, -5.9540e-01, -2.9808e-02, 3.1468e-01, -4.5809e-01, 4.3313e-03 ], [ 1.2234e-01, -1.3285e-01, 6.3068e-02, -2.3343e-02, -2.3519e-01, -4.0794e-01, -5.6063e-01, -5.5484e-01, -1.1272e-01, 3.0103e-01, -4.0983e-01, 3.3038e-02 ]], [[ 9.8597e-02, -1.2121e-01, 1.0718e-01, -2.2644e-02, -4.0282e-01, -4.2646e-01, -5.9981e-01, -3.7200e-01, 1.9538e-01, 2.7036e-01, -3.4072e-01, -1.7966e-03 ], [ 8.8470e-02, -1.2618e-01, 5.3351e-02, -1.8531e-02, -3.3834e-01, -4.9047e-01, -5.7063e-01, -4.9790e-01, 2.2070e-01, 3.3964e-01, -4.1604e-01, 2.3519e-02 ], [ 5.8373e-02, -1.2706e-01, 1.0598e-01, 9.3277e-05, -3.0493e-01, -4.4406e-01, -5.4723e-01, -5.2214e-01, 8.0374e-02, 2.6307e-01, -4.4571e-01, 8.7052e-02 ], [ 7.9567e-02, -1.2977e-01, 1.1731e-01, 2.6198e-02, -2.4024e-01, -4.2161e-01, -5.7604e-01, -7.3298e-01, 1.6698e-01, 3.1454e-01, -4.9189e-01, 2.4027e-02 ]]]) self.assertTensorAlmostEqual(output_target, output)