def __init__(self, config, num_labels):
super(SequenceSummary, self).__init__()
self.summary = nn.Dense(config.d_model,
num_labels,
weight_init=weight_variable(
[config.d_model, num_labels]),
has_bias=True).to_float(config.compute_type)
self.gelu = nn.GELU()
self.first_dropout = nn.Dropout(1 - config.hidden_dropout)
self.last_dropout = nn.Dropout(1 - config.hidden_dropout)
self.expand_dims = P.ExpandDims()
self.shape = P.Shape()
self.size = P.Size()
self.slice = P.GatherV2()
self.squeeze = P.Squeeze(-2)
def orthogonal(tensor, gain=1):
r"""
Description:
Fills the input `Tensor` with a (semi) orthogonal matrix, as
described in `Exact solutions to the nonlinear dynamics of learning in deep
linear neural networks` - Saxe, A. et al. (2013). The input tensor must have
at least 2 dimensions, and for tensors with more than 2 dimensions the
trailing dimensions are flattened.
Args:
tensor: an n-dimensional `mindspore.Tensor`, where :math:`n \geq 2`
gain: optional scaling factor
Examples:
>>> w = Tensor(np.empty([3, 5]), mindspore.float32)
>>> orthogonal(w)
"""
size_op = P.Size()
transpose = P.Transpose()
mul = P.Mul()
if len(tensor.shape) < 2:
raise ValueError("Only tensors with 2 or more dimensions are supported")
rows = tensor.shape[0]
cols = size_op(tensor) // rows
flatten = Tensor(np.random.normal(size=(rows, cols)), mindspore.float32 )
if rows < cols:
flatten = transpose(flatten, (1, 0))
#compute the qr factorization
q, r = np.linalg.qr(flatten.asnumpy())
d = np.diag(r, 0)
ph = np.sign(d)
q *= ph
q = Tensor(q, mindspore.float32)
if rows < cols:
q = transpose(q, (1, 0))
q = mul(q, gain)
return q
def __init__(self,
probs=None,
logits=None,
seed=None,
dtype=mstype.int32,
name="Categorical"):
param = dict(locals())
valid_dtype = mstype.int_type
check_type(dtype, valid_dtype, "Categorical")
super(Categorical, self).__init__(seed, dtype, name, param)
if (probs is None) == (logits is None):
raise_probs_logits_error()
self.reduce_sum = P.ReduceSum(keep_dims=True)
self.reduce_sum1 = P.ReduceSum(keep_dims=False)
self.log = P.Log()
self.exp = P.Exp()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.div = P.RealDiv()
self.size = P.Size()
self.mutinomial = P.Multinomial(seed=self.seed)
self.cast = P.Cast()
self.expandim = P.ExpandDims()
self.gather = P.GatherNd()
self.concat = P.Concat(-1)
self.transpose = P.Transpose()
if probs is not None:
self._probs = cast_to_tensor(probs, mstype.float32)
input_sum = self.reduce_sum(self._probs, -1)
self._probs = self.div(self._probs, input_sum)
self._logits = probs_to_logits(self._probs)
self._param = self._probs
else:
self._logits = cast_to_tensor(logits, mstype.float32)
input_sum = self.reduce_sum(self.exp(self._logits), -1)
self._logits = self._logits - self.log(input_sum)
self._probs = logits_to_probs(self._logits)
self._param = self._logits
self._num_events = self.shape(self._param)[-1]
self._param2d = self.reshape(self._param, (-1, self._num_events))
self._batch_shape = self.shape(self._param)[:-1]
self._batch_shape_n = (1, ) * len(self._batch_shape)
def __init__(self,
probs=None,
logits=None,
seed=0,
dtype=mstype.int32,
name="Categorical"):
param = dict(locals())
super(Categorical, self).__init__(seed, dtype, name, param)
if (probs is None) == (logits is None):
raise ValueError(
"Either 'prob' or 'logits' must be specified, but not both.")
self.reduce_sum = P.ReduceSum(keep_dims=True)
self.log = P.Log()
self.exp = P.Exp()
self.shape = P.Shape()
self.reshape = P.Reshape()
self.div = P.RealDiv()
self.size = P.Size()
self.mutinomial = P.Multinomial(seed=seed)
self.cast = P.Cast()
self.expandim = P.ExpandDims()
self.gather = P.GatherNd()
self.concat = P.Concat(-1)
if probs is not None:
self._probs = cast_to_tensor(probs, mstype.float32)
input_sum = self.reduce_sum(self._probs, -1)
self._probs = self.div(self._probs, input_sum)
self._logits = probs_to_logits(self._probs)
self._param = self._probs
else:
self._logits = cast_to_tensor(logits, mstype.float32)
input_sum = self.reduce_sum(self.exp(self._logits), -1)
self._logits = self._logits - self.log(input_sum)
self._probs = logits_to_probs(self._logits)
self._param = self._logits
self._num_events = self.shape(self._param)[-1]
self._param2d = self.reshape(self._param, (-1, self._num_events))
self._batch_shape = self.shape(self._param2d)[0]
'desc_bprop': [[3, 2]]}),
('Squeeze_0', {
'block': P.Squeeze(),
'desc_inputs': [[3, 1, 2, 1]],
'desc_bprop': [[3, 2]]}),
('Squeeze_1', {
'block': P.Squeeze(),
'desc_inputs': [[1, 1, 1, 1]],
'desc_bprop': [1.0],
'skip': ['backward']}),
('Squeeze_2', {
'block': P.Squeeze((2, 3)),
'desc_inputs': [[3, 2, 1, 1]],
'desc_bprop': [[3, 2]]}),
('Size', {
'block': P.Size(),
'desc_inputs': [[2, 3, 5]],
'skip': ['backward']}),
('Tile_0', {
'block': P.Tile(),
'desc_const': [(1, 2)],
'desc_inputs': [[64, 1]],
'desc_bprop': [[64, 2]]}),
('Tile_1', {
'block': P.Tile(),
'desc_const': [(1, 1)],
'desc_inputs': [[64, 1]],
'desc_bprop': [[64, 1]]}),
('Tile_2', {
'block': P.Tile(),
'desc_const': [(2, 1, 1, 2)],
from mindspore._checkparam import Validator as validator
from mindspore._checkparam import Rel
from mindspore.nn import Optimizer
from mindspore.nn import TrainOneStepCell, WithLossCell
from mindspore.nn.optim import Momentum
from mindspore.train import Model
from ....dataset_mock import MindData
context.set_context(mode=context.GRAPH_MODE, enable_sparse=True)
reduce_sum = P.ReduceSum()
unsorted_segment_sum = P.UnsortedSegmentSum()
transpose = P.Transpose()
shape_op = P.Shape()
reshape = P.Reshape()
size_op = P.Size()
invert_permutation = P.InvertPermutation()
logical_and = P.LogicalAnd()
def get_axis(x):
shape = shape_op(x)
length = F.tuple_len(shape)
perm = F.make_range(0, length)
return perm
class MSELoss(nn.Cell):
def __init__(self):
super(MSELoss, self).__init__()
self.reduce_sum = P.ReduceSum()
