def testBiasFuture(self):
"""Testing the sequence order mask."""
q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32)
q = tf.expand_dims(q, axis=-1)
k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32)
k = tf.expand_dims(k, axis=-1)
ground_truth = np.array(
[
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
[0, 0, 0, 1, 1, 0, 0, 0], # 2
[0, 0, 0, 0, 1, 0, 0, 0], # 3
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
[0, 0, 0, 1, 1, 0, 0, 0], # 2
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
],
np.float32) * -1e9
bias = common_attention.attention_bias_future(q, k)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
self.assertAllClose(
bias.eval(),
ground_truth,
)
def testBiasFuture(self):
"""Testing the sequence order mask."""
q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32)
q = tf.expand_dims(q, axis=-1)
k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32)
k = tf.expand_dims(k, axis=-1)
ground_truth = np.array([
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
[0, 0, 0, 1, 1, 0, 0, 0], # 2
[0, 0, 0, 0, 1, 0, 0, 0], # 3
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
[0, 0, 0, 1, 1, 0, 0, 0], # 2
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
], np.float32) * -1e9
bias = common_attention.attention_bias_future(q, k)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
self.assertAllClose(
bias.eval(),
ground_truth,
)
def testBiasFuture(self):
"""Testing the sequence order mask."""
q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32)
q = tf.expand_dims(q, axis=-1)
k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32)
k = tf.expand_dims(k, axis=-1)
ground_truth = np.array([
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
[0, 0, 0, 1, 1, 0, 0, 0], # 2
[0, 0, 0, 0, 1, 0, 0, 0], # 3
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
[0, 0, 0, 1, 1, 0, 0, 0], # 2
[0, 1, 1, 1, 1, 0, 1, 1], # 0
[0, 0, 1, 1, 1, 0, 0, 1], # 1
], np.float32) * -1e9
bias = common_attention.attention_bias_future(q, k)
self.assertAllClose(self.evaluate(bias), ground_truth)