def __init__(self, sparse=False):
super(SoftmaxCrossEntropyExpand, self).__init__()
self.exp = ops.Exp()
self.sum = ops.ReduceSum(keep_dims=True)
self.onehot = ops.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.div = ops.RealDiv()
self.log = ops.Log()
self.sum_cross_entropy = ops.ReduceSum(keep_dims=False)
self.mul = ops.Mul()
self.mul2 = ops.Mul()
self.mean = ops.ReduceMean(keep_dims=False)
self.sparse = sparse
self.max = ops.ReduceMax(keep_dims=True)
self.sub = ops.Sub()
def ranking_loss(self, input1, input2, y):
sub = P.Sub()
mul = P.Mul()
add = P.Add()
temp1 = -sub(input1, input2)
temp2 = mul(temp1, y)
temp3 = add(temp2, self.margin)
temp3_mask = np.greater_equal(temp3, 0)
loss = 0
for i in range(temp3.shape[0]):
if temp3_mask[i]:
loss += temp3[i]
loss = Tensor(loss / temp3.shape[0])
# print(loss)
return loss
def __init__(self, config, is_training, num_tokens, dropout_prob=0.0, use_one_hot_embeddings=False):
super(BertPoetryModel, self).__init__()
self.bert = BertModel(config, is_training, use_one_hot_embeddings)
self.num_tokens = num_tokens
idx = np.arange(config.seq_length)
mask = idx[None, :] <= idx[:, None]
self.mask = Tensor([mask], mstype.float32)
self.MLM_Dense = nn.Dense(config.hidden_size, config.hidden_size,\
has_bias=True, weight_init=TruncatedNormal(0.02),\
activation='gelu').to_float(mstype.float16)
self.layer_norm = nn.LayerNorm((config.hidden_size,))
self.matmul = ops.MatMul(transpose_b=True)
self.biasadd = Parameter(initializer('zero', self.num_tokens), name='MLM_output_biasadd')
self.softmax = ops.Softmax(axis=-1)
self.seq_length = config.seq_length
self.hidden_size = config.hidden_size
self.cast = ops.Cast()
self.reshape = ops.Reshape()
self.batch_matmul = ops.BatchMatMul()
ones = np.ones(shape=(config.batch_size, config.seq_length, config.seq_length))
self.lower_triangle_mask = Tensor(np.tril(ones), dtype=mstype.float32)
self.multiply = ops.Mul()
def __init__(self,
batch_size,
from_tensor_width,
to_tensor_width,
from_seq_length,
to_seq_length,
num_attention_heads=1,
size_per_head=512,
query_act=None,
key_act=None,
value_act=None,
has_attention_mask=False,
attention_probs_dropout_prob=0.0,
use_one_hot_embeddings=False,
initializer_range=0.02,
do_return_2d_tensor=False,
use_relative_positions=False,
compute_type=mstype.float32):
super(BertAttention, self).__init__()
self.batch_size = batch_size
self.from_seq_length = from_seq_length
self.to_seq_length = to_seq_length
self.num_attention_heads = num_attention_heads
self.size_per_head = size_per_head
self.has_attention_mask = has_attention_mask
self.use_relative_positions = use_relative_positions
self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))], dtype=compute_type)
self.reshape = ops.Reshape()
self.shape_from_2d = (-1, from_tensor_width)
self.shape_to_2d = (-1, to_tensor_width)
weight = TruncatedNormal(initializer_range)
units = num_attention_heads * size_per_head
self.query_layer = nn.Dense(from_tensor_width,
units,
activation=query_act,
weight_init=weight).to_float(compute_type)
self.key_layer = nn.Dense(to_tensor_width,
units,
activation=key_act,
weight_init=weight).to_float(compute_type)
self.value_layer = nn.Dense(to_tensor_width,
units,
activation=value_act,
weight_init=weight).to_float(compute_type)
self.shape_from = (batch_size, from_seq_length, num_attention_heads, size_per_head)
self.shape_to = (
batch_size, to_seq_length, num_attention_heads, size_per_head)
self.matmul_trans_b = ops.BatchMatMul(transpose_b=True)
self.multiply = ops.Mul()
self.transpose = ops.Transpose()
self.trans_shape = (0, 2, 1, 3)
self.trans_shape_relative = (2, 0, 1, 3)
self.trans_shape_position = (1, 2, 0, 3)
#self.multiply_data = Tensor([-10000.0,], dtype=compute_type)
self.multiply_data = Tensor([-10000.0,], dtype=mstype.float32)
self.batch_num = batch_size * num_attention_heads
self.matmul = ops.BatchMatMul()
self.softmax = nn.Softmax()
self.dropout = nn.Dropout(1 - attention_probs_dropout_prob)
if self.has_attention_mask:
self.expand_dims = ops.ExpandDims()
self.sub = ops.Sub()
self.add = ops.TensorAdd()
self.cast = ops.Cast()
self.get_dtype = ops.DType()
if do_return_2d_tensor:
self.shape_return = (batch_size * from_seq_length, num_attention_heads * size_per_head)
else:
self.shape_return = (batch_size, from_seq_length, num_attention_heads * size_per_head)
self.cast_compute_type = SaturateCast(dst_type=compute_type)
if self.use_relative_positions:
self._generate_relative_positions_embeddings = \
RelaPosEmbeddingsGenerator(length=to_seq_length,
depth=size_per_head,
max_relative_position=16,
initializer_range=initializer_range,
use_one_hot_embeddings=use_one_hot_embeddings)
def __init__(self):
super(Net, self).__init__()
self.mul = ops.Mul()
def construct(self, inputs, targets):
"""
Args:
- inputs: feature matrix with shape (batch_size, feat_dim)
- targets: ground truth labels with shape (num_classes)
"""
n = inputs.shape[0]
# Compute pairwise distance, replace by the official when merged
pow = P.Pow()
sum = P.ReduceSum(keep_dims=True)
expand = P.BroadcastTo((n, n))
transpose = P.Transpose()
mul = P.Mul()
add = P.Add()
sqrt = P.Sqrt()
equal = P.Equal()
cat = P.Concat()
ones_like = P.OnesLike()
dist = pow(inputs, 2)
dist = sum(dist, axis=1)
dist = expand(dist)
dist = dist + transpose(dist, (1, 0))
temp1 = P.matmul(inputs, transpose(inputs, (1, 0)))
temp1 = mul(-2, temp1)
dist = add(dist, temp1)
dist = P.composite.clip_by_value(
dist, clip_value_min=1e-12, clip_value_max=100000000
) # for numerical stability, clip_value_max=? why must set?
dist = sqrt(dist)
# For each anchor, find the hardest positive and negative
targets = expand(targets)
mask = equal(targets, transpose(targets, (1, 0)))
dist_ap = []
dist_an = []
# only for debugging
#####################
# print("dist is")
# print(dist.shape)
# print(dist)
# print("mask is")
# print(mask.shape)
# print(mask)
# print(mask[0])
#####################
for i in range(n):
minval = -1.0
maxval = -1.0
for j in range(n):
if mask[i][j] and dist[i][j] > maxval:
maxval = dist[i][j]
if not mask[i][j] and (dist[i][j] < minval or minval == -1):
minval = dist[i][j]
if (not isinstance(minval, Tensor)
or not isinstance(maxval, Tensor) or minval == -1.0
or maxval == -1.0):
if self.error_msg is not None:
print("Error Msg", file=self.error_msg)
print("mask {} is".format(i), file=self.error_msg)
print(mask[i], file=self.error_msg)
print("dist is:", file=self.error_msg)
print(dist[i], file=self.error_msg)
print(maxval, file=self.error_msg)
print(minval, file=self.error_msg)
print(type(maxval), file=self.error_msg)
print(type(minval), file=self.error_msg)
self.error_msg.flush()
# assert minval != -1.0 and isinstance(minval, Tensor)
# assert maxval != -1.0 and isinstance(maxval, Tensor)
dist_ap.append(maxval.asnumpy())
dist_an.append(minval.asnumpy())
dist_ap = Tensor(dist_ap, ms.float32)
dist_an = Tensor(dist_an, ms.float32)
# only for debugging
#####################
# print(dist_ap)
# print(dist_ap.shape)
# print(dist_an)
#####################
# Compute ranking hinge loss
y = ones_like(dist_an)
loss = self.ranking_loss(dist_an, dist_ap, y)
# # compute accuracy
# correct = torch.ge(dist_an, dist_ap).sum().item()
return loss
# class GradOriTripletLoss(nn.Cell)
# def __init__(self, net):
# super(GradOriTripletLoss, self).__init__()
# self.net = net
# self.grad_op = P.GradOperation(get_all=True)
#
# def construct(self, inputs, targets):
# gradient_function = self.grad_op(self.net)
# return gradient_function(inputs, targets)
def __init__(self):
super(Net, self).__init__()
self.mul = ops.Mul()
weight_np = np.array([2, 2]).astype(np.float32)
self.weight = Parameter(Tensor(weight_np), name="weight", requires_grad=True)
