def __init__(self,
batch_size,
from_seq_length,
to_seq_length,
num_attention_heads=1,
size_per_head=512,
use_one_hot_embeddings=False,
initializer_range=0.02,
do_return_2d_tensor=False,
use_relative_positions=False,
dtype=mstype.float32,
compute_type=mstype.float32):
super(BertAttentionRelativePositionValues, self).__init__()
self.batch_size = batch_size
self.from_seq_length = from_seq_length
self.to_seq_length = to_seq_length
self.use_relative_positions = use_relative_positions
self.size_per_head = size_per_head
self.num_attention_heads = num_attention_heads
self.trans_shape_position = (1, 2, 0, 3)
self.trans_shape_relative = (2, 0, 1, 3)
self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))],
dtype=dtype)
self.trans_shape = (0, 2, 1, 3)
self.reshape = P.Reshape()
self.multiply = P.Mul()
self.transpose = P.Transpose()
self.batch_num = batch_size * num_attention_heads
self.matmul = P.BatchMatMul()
self.do_return_2d_tensor = do_return_2d_tensor
if self.do_return_2d_tensor:
self.shp_return = (batch_size * from_seq_length,
num_attention_heads * size_per_head)
else:
self.shp_return = (batch_size, from_seq_length,
num_attention_heads * size_per_head)
self.cast_compute_type = SaturateCast(dst_type=compute_type)
self._generate_relative_positions_embeddings = \
RelaPosEmbeddingsGenerator(length=self.to_seq_length,
depth=self.size_per_head,
max_relative_position=16,
initializer_range=initializer_range,
use_one_hot_embeddings=use_one_hot_embeddings)
self.fill = P.Fill()
self.multiply = P.Mul()
self.type = P.DType()
self.cast = P.Cast()
def __init__(self,
batch_size,
from_seq_length,
to_seq_length,
num_attention_heads=1,
size_per_head=512,
use_one_hot_embeddings=False,
initializer_range=0.02,
use_relative_positions=False,
dtype=mstype.float32,
compute_type=mstype.float32):
super(BertAttentionRelativePositionKeys, self).__init__()
self.batch_size = batch_size
self.from_seq_length = from_seq_length
self.to_seq_length = to_seq_length
self.use_relative_positions = use_relative_positions
self.size_per_head = size_per_head
self.num_attention_heads = num_attention_heads
self.trans_shape_position = (1, 2, 0, 3)
self.trans_shape_relative = (2, 0, 1, 3)
self.scores_mul = 1.0 / math.sqrt(float(self.size_per_head))
self.reshape = P.Reshape()
self.multiply = P.Mul()
self.transpose = P.Transpose()
self.matmul_trans_b = P.BatchMatMul(transpose_b=True)
self.batch_num = batch_size * num_attention_heads
self.cast = P.Cast()
self.cast_compute_type = SaturateCast(dst_type=compute_type)
self._generate_relative_positions_embeddings = \
RelaPosEmbeddingsGenerator(length=self.to_seq_length,
depth=self.size_per_head,
max_relative_position=16,
initializer_range=initializer_range,
use_one_hot_embeddings=use_one_hot_embeddings)
}, {
'id':
'RelaPosMatrixGenerator',
'group':
'RelaPosMatrixGenerator',
'block':
RelaPosMatrixGenerator(length=128, max_relative_position=16)
}, {
'id':
'RelaPosEmbeddingsGenerator',
'group':
'RelaPosEmbeddingsGenerator',
'block':
RelaPosEmbeddingsGenerator(length=128,
depth=64,
max_relative_position=16,
initializer_range=0.02,
use_one_hot_embeddings=False)
}, {
'id':
'BertAttention',
'group':
'BertAttention',
'block':
BertAttention(batch_size=1,
from_tensor_width=1024,
to_tensor_width=1024,
from_seq_length=128,
to_seq_length=128,
num_attention_heads=16,
size_per_head=64,