def test_scaled_dot_product_attention5(): """ value different dimension to query and key, AND query and key can have different sequence length, AND key and value must have same sequence length """ s1, s2 = 4, 2 s3, s4 = 5, 10 seq1 = C.Axis.new_unique_dynamic_axis('seq1') seq2 = C.Axis.new_unique_dynamic_axis('seq2') query = C.sequence.input_variable(5, sequence_axis=seq1) key = C.sequence.input_variable(5, sequence_axis=seq2) value = C.sequence.input_variable(7, sequence_axis=seq2) b = ScaledDotProductAttention()(query, key, value) q1 = np.random.random((s3, 5)).astype(np.float32) q2 = np.random.random((s4, 5)).astype(np.float32) k1 = np.random.random((s1, 5)).astype(np.float32) k2 = np.random.random((s2, 5)).astype(np.float32) v1 = np.random.random((s1, 7)).astype(np.float32) v2 = np.random.random((s2, 7)).astype(np.float32) results = b.eval({query: [q1, q2], key: [k1, k2], value: [v1, v2]})
def test_scaled_dot_product_attention6(): """ key and value must have same sequence length """ s1, s2 = 4, 2 s3, s4 = 5, 10 seq1 = C.Axis.new_unique_dynamic_axis('seq1') seq2 = C.Axis.new_unique_dynamic_axis('seq2') query = C.sequence.input_variable(5, sequence_axis=seq1) key = C.sequence.input_variable(5, sequence_axis=seq1) value = C.sequence.input_variable(7, sequence_axis=seq2) b = ScaledDotProductAttention()(query, key, value) q1 = np.random.random((s3, 5)).astype(np.float32) q2 = np.random.random((s4, 5)).astype(np.float32) k1 = np.random.random((s3, 5)).astype(np.float32) k2 = np.random.random((s4, 5)).astype(np.float32) v1 = np.random.random((s1, 7)).astype(np.float32) v2 = np.random.random((s2, 7)).astype(np.float32) with pytest.raises(Exception): results = b.eval({query: [q1, q2], key: [k1, k2], value: [v1, v2]})
def test_scaled_dot_product_attention1(): """ check default works """ s1, s2 = 4, 2 a = C.sequence.input_variable(5) b = ScaledDotProductAttention()(a, a, a) assert b.shape == (5, ), "output should be a sequence and dimension should not change" n1 = np.random.random((s1, 5)).astype(np.float32) n2 = np.random.random((s2, 5)).astype(np.float32) results = b.eval({a: [n1, n2]}) assert results[1].shape == n2.shape, f"Wrong expected shape {results[1].shape} != {n2.shape}" assert results[0].shape == n1.shape, f"Wrong expected shape {results[0].shape} != {n1.shape}"
def test_scaled_dot_product_attention2(): """ returns a sequence while not peeking on future values """ s1, s2 = 4, 2 a = C.sequence.input_variable(5) b = ScaledDotProductAttention(obey_sequence_order=True, max_seq_len=100)(a, a, a) assert b.shape == (5, ), "output should be a sequence and dimension should not change" n1 = np.random.random((s1, 5)).astype(np.float32) n2 = np.random.random((s2, 5)).astype(np.float32) results = b.eval({a: [n1, n2]}) assert results[1].shape == n2.shape, f"Wrong expected shape {results[1].shape} != {n2.shape}" assert results[0].shape == n1.shape, f"Wrong expected shape {results[0].shape} != {n1.shape}"
def test_scaled_dot_product_attention4(): """ value can be of a completely different dimensions as the query or key """ s1, s2 = 4, 2 query = C.sequence.input_variable(5) key = C.sequence.input_variable(5) value = C.sequence.input_variable(7) b = ScaledDotProductAttention()(query, key, value) n1 = np.random.random((s1, 5)).astype(np.float32) n2 = np.random.random((s2, 5)).astype(np.float32) m1 = np.random.random((s1, 7)).astype(np.float32) m2 = np.random.random((s2, 7)).astype(np.float32) results = b.eval({query: [n1, n2], key: [n1, n2], value: [m1, m2]})