def __init__(self, config, num_labels): super(SequenceSummary, self).__init__() self.summary = nn.Dense(config.d_model, num_labels, weight_init=weight_variable( [config.d_model, num_labels]), has_bias=True).to_float(config.compute_type) self.gelu = nn.GELU() self.first_dropout = nn.Dropout(1 - config.hidden_dropout) self.last_dropout = nn.Dropout(1 - config.hidden_dropout) self.expand_dims = P.ExpandDims() self.shape = P.Shape() self.size = P.Size() self.slice = P.GatherV2() self.squeeze = P.Squeeze(-2)
def orthogonal(tensor, gain=1): r""" Description: Fills the input `Tensor` with a (semi) orthogonal matrix, as described in `Exact solutions to the nonlinear dynamics of learning in deep linear neural networks` - Saxe, A. et al. (2013). The input tensor must have at least 2 dimensions, and for tensors with more than 2 dimensions the trailing dimensions are flattened. Args: tensor: an n-dimensional `mindspore.Tensor`, where :math:`n \geq 2` gain: optional scaling factor Examples: >>> w = Tensor(np.empty([3, 5]), mindspore.float32) >>> orthogonal(w) """ size_op = P.Size() transpose = P.Transpose() mul = P.Mul() if len(tensor.shape) < 2: raise ValueError("Only tensors with 2 or more dimensions are supported") rows = tensor.shape[0] cols = size_op(tensor) // rows flatten = Tensor(np.random.normal(size=(rows, cols)), mindspore.float32 ) if rows < cols: flatten = transpose(flatten, (1, 0)) #compute the qr factorization q, r = np.linalg.qr(flatten.asnumpy()) d = np.diag(r, 0) ph = np.sign(d) q *= ph q = Tensor(q, mindspore.float32) if rows < cols: q = transpose(q, (1, 0)) q = mul(q, gain) return q
def __init__(self, probs=None, logits=None, seed=None, dtype=mstype.int32, name="Categorical"): param = dict(locals()) valid_dtype = mstype.int_type check_type(dtype, valid_dtype, "Categorical") super(Categorical, self).__init__(seed, dtype, name, param) if (probs is None) == (logits is None): raise_probs_logits_error() self.reduce_sum = P.ReduceSum(keep_dims=True) self.reduce_sum1 = P.ReduceSum(keep_dims=False) self.log = P.Log() self.exp = P.Exp() self.shape = P.Shape() self.reshape = P.Reshape() self.div = P.RealDiv() self.size = P.Size() self.mutinomial = P.Multinomial(seed=self.seed) self.cast = P.Cast() self.expandim = P.ExpandDims() self.gather = P.GatherNd() self.concat = P.Concat(-1) self.transpose = P.Transpose() if probs is not None: self._probs = cast_to_tensor(probs, mstype.float32) input_sum = self.reduce_sum(self._probs, -1) self._probs = self.div(self._probs, input_sum) self._logits = probs_to_logits(self._probs) self._param = self._probs else: self._logits = cast_to_tensor(logits, mstype.float32) input_sum = self.reduce_sum(self.exp(self._logits), -1) self._logits = self._logits - self.log(input_sum) self._probs = logits_to_probs(self._logits) self._param = self._logits self._num_events = self.shape(self._param)[-1] self._param2d = self.reshape(self._param, (-1, self._num_events)) self._batch_shape = self.shape(self._param)[:-1] self._batch_shape_n = (1, ) * len(self._batch_shape)
def __init__(self, probs=None, logits=None, seed=0, dtype=mstype.int32, name="Categorical"): param = dict(locals()) super(Categorical, self).__init__(seed, dtype, name, param) if (probs is None) == (logits is None): raise ValueError( "Either 'prob' or 'logits' must be specified, but not both.") self.reduce_sum = P.ReduceSum(keep_dims=True) self.log = P.Log() self.exp = P.Exp() self.shape = P.Shape() self.reshape = P.Reshape() self.div = P.RealDiv() self.size = P.Size() self.mutinomial = P.Multinomial(seed=seed) self.cast = P.Cast() self.expandim = P.ExpandDims() self.gather = P.GatherNd() self.concat = P.Concat(-1) if probs is not None: self._probs = cast_to_tensor(probs, mstype.float32) input_sum = self.reduce_sum(self._probs, -1) self._probs = self.div(self._probs, input_sum) self._logits = probs_to_logits(self._probs) self._param = self._probs else: self._logits = cast_to_tensor(logits, mstype.float32) input_sum = self.reduce_sum(self.exp(self._logits), -1) self._logits = self._logits - self.log(input_sum) self._probs = logits_to_probs(self._logits) self._param = self._logits self._num_events = self.shape(self._param)[-1] self._param2d = self.reshape(self._param, (-1, self._num_events)) self._batch_shape = self.shape(self._param2d)[0]
'desc_bprop': [[3, 2]]}), ('Squeeze_0', { 'block': P.Squeeze(), 'desc_inputs': [[3, 1, 2, 1]], 'desc_bprop': [[3, 2]]}), ('Squeeze_1', { 'block': P.Squeeze(), 'desc_inputs': [[1, 1, 1, 1]], 'desc_bprop': [1.0], 'skip': ['backward']}), ('Squeeze_2', { 'block': P.Squeeze((2, 3)), 'desc_inputs': [[3, 2, 1, 1]], 'desc_bprop': [[3, 2]]}), ('Size', { 'block': P.Size(), 'desc_inputs': [[2, 3, 5]], 'skip': ['backward']}), ('Tile_0', { 'block': P.Tile(), 'desc_const': [(1, 2)], 'desc_inputs': [[64, 1]], 'desc_bprop': [[64, 2]]}), ('Tile_1', { 'block': P.Tile(), 'desc_const': [(1, 1)], 'desc_inputs': [[64, 1]], 'desc_bprop': [[64, 1]]}), ('Tile_2', { 'block': P.Tile(), 'desc_const': [(2, 1, 1, 2)],
from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel from mindspore.nn import Optimizer from mindspore.nn import TrainOneStepCell, WithLossCell from mindspore.nn.optim import Momentum from mindspore.train import Model from ....dataset_mock import MindData context.set_context(mode=context.GRAPH_MODE, enable_sparse=True) reduce_sum = P.ReduceSum() unsorted_segment_sum = P.UnsortedSegmentSum() transpose = P.Transpose() shape_op = P.Shape() reshape = P.Reshape() size_op = P.Size() invert_permutation = P.InvertPermutation() logical_and = P.LogicalAnd() def get_axis(x): shape = shape_op(x) length = F.tuple_len(shape) perm = F.make_range(0, length) return perm class MSELoss(nn.Cell): def __init__(self): super(MSELoss, self).__init__() self.reduce_sum = P.ReduceSum()