Python attention_bias_future Examples

Example #1

File: common_attention_test.py Project: vplab-github/transformer-caption

    def testBiasFuture(self):
        """Testing the sequence order mask."""
        q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32)
        q = tf.expand_dims(q, axis=-1)

        k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32)
        k = tf.expand_dims(k, axis=-1)

        ground_truth = np.array(
            [
                [0, 1, 1, 1, 1, 0, 1, 1],  # 0
                [0, 0, 1, 1, 1, 0, 0, 1],  # 1
                [0, 0, 0, 1, 1, 0, 0, 0],  # 2
                [0, 0, 0, 0, 1, 0, 0, 0],  # 3
                [0, 1, 1, 1, 1, 0, 1, 1],  # 0
                [0, 0, 1, 1, 1, 0, 0, 1],  # 1
                [0, 0, 0, 1, 1, 0, 0, 0],  # 2
                [0, 1, 1, 1, 1, 0, 1, 1],  # 0
                [0, 0, 1, 1, 1, 0, 0, 1],  # 1
            ],
            np.float32) * -1e9

        bias = common_attention.attention_bias_future(q, k)

        with self.test_session() as session:
            session.run(tf.global_variables_initializer())
            self.assertAllClose(
                bias.eval(),
                ground_truth,
            )

Example #2

File: common_attention_test.py Project: AranKomat/tensor2tensor

  def testBiasFuture(self):
    """Testing the sequence order mask."""
    q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32)
    q = tf.expand_dims(q, axis=-1)

    k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32)
    k = tf.expand_dims(k, axis=-1)

    ground_truth = np.array([
        [0, 1, 1, 1, 1, 0, 1, 1],  # 0
        [0, 0, 1, 1, 1, 0, 0, 1],  # 1
        [0, 0, 0, 1, 1, 0, 0, 0],  # 2
        [0, 0, 0, 0, 1, 0, 0, 0],  # 3
        [0, 1, 1, 1, 1, 0, 1, 1],  # 0
        [0, 0, 1, 1, 1, 0, 0, 1],  # 1
        [0, 0, 0, 1, 1, 0, 0, 0],  # 2
        [0, 1, 1, 1, 1, 0, 1, 1],  # 0
        [0, 0, 1, 1, 1, 0, 0, 1],  # 1
    ], np.float32) * -1e9

    bias = common_attention.attention_bias_future(q, k)

    with self.test_session() as session:
      session.run(tf.global_variables_initializer())
      self.assertAllClose(
          bias.eval(),
          ground_truth,
      )

Example #3

  def testBiasFuture(self):
    """Testing the sequence order mask."""
    q = tf.constant([0, 1, 2, 3, 0, 1, 2, 0, 1], dtype=tf.int32)
    q = tf.expand_dims(q, axis=-1)

    k = tf.constant([0, 1, 2, 3, 4, 0, 1, 2], dtype=tf.int32)
    k = tf.expand_dims(k, axis=-1)

    ground_truth = np.array([
        [0, 1, 1, 1, 1, 0, 1, 1],  # 0
        [0, 0, 1, 1, 1, 0, 0, 1],  # 1
        [0, 0, 0, 1, 1, 0, 0, 0],  # 2
        [0, 0, 0, 0, 1, 0, 0, 0],  # 3
        [0, 1, 1, 1, 1, 0, 1, 1],  # 0
        [0, 0, 1, 1, 1, 0, 0, 1],  # 1
        [0, 0, 0, 1, 1, 0, 0, 0],  # 2
        [0, 1, 1, 1, 1, 0, 1, 1],  # 0
        [0, 0, 1, 1, 1, 0, 0, 1],  # 1
    ], np.float32) * -1e9

    bias = common_attention.attention_bias_future(q, k)
    self.assertAllClose(self.evaluate(bias), ground_truth)