def testBiasFuture(self): """Testing the sequence order mask.""" q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32) q = tf.expand_dims(q, axis=-1) k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32) k = tf.expand_dims(k, axis=-1) ground_truth = np.array( [ [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 [0, 0, 0, 1, 1, 0, 0, 0], # 2 [0, 0, 0, 0, 1, 0, 0, 0], # 3 [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 [0, 0, 0, 1, 1, 0, 0, 0], # 2 [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 ], np.float32) * -1e9 bias = common_attention.attention_bias_future(q, k) with self.test_session() as session: session.run(tf.global_variables_initializer()) self.assertAllClose( bias.eval(), ground_truth, )
def testBiasFuture(self): """Testing the sequence order mask.""" q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32) q = tf.expand_dims(q, axis=-1) k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32) k = tf.expand_dims(k, axis=-1) ground_truth = np.array([ [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 [0, 0, 0, 1, 1, 0, 0, 0], # 2 [0, 0, 0, 0, 1, 0, 0, 0], # 3 [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 [0, 0, 0, 1, 1, 0, 0, 0], # 2 [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 ], np.float32) * -1e9 bias = common_attention.attention_bias_future(q, k) with self.test_session() as session: session.run(tf.global_variables_initializer()) self.assertAllClose( bias.eval(), ground_truth, )
def testBiasFuture(self): """Testing the sequence order mask.""" q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32) q = tf.expand_dims(q, axis=-1) k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32) k = tf.expand_dims(k, axis=-1) ground_truth = np.array([ [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 [0, 0, 0, 1, 1, 0, 0, 0], # 2 [0, 0, 0, 0, 1, 0, 0, 0], # 3 [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 [0, 0, 0, 1, 1, 0, 0, 0], # 2 [0, 1, 1, 1, 1, 0, 1, 1], # 0 [0, 0, 1, 1, 1, 0, 0, 1], # 1 ], np.float32) * -1e9 bias = common_attention.attention_bias_future(q, k) self.assertAllClose(self.evaluate(bias), ground_truth)