def __init__(self, normalized_shape, eps=1e-5): super(LayerNorm, self).__init__() self.gamma = Parameter(initializer('ones', normalized_shape), name="gamma") self.beta = Parameter(initializer('zeros', normalized_shape), name="beta") self.mean = P.ReduceMean(keep_dims=True) self.eps = eps self.sub = P.Sub() self.add = P.Add() self.mul = P.Mul() self.div = P.RealDiv() self.reshape = P.Reshape() self.shape = P.Shape()
def __init__(self, num_cls=21, ignore_label=255): super(SoftmaxCrossEntropyLoss, self).__init__() self.one_hot = P.OneHot(axis=-1) self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.cast = P.Cast() self.ce = nn.SoftmaxCrossEntropyWithLogits() self.not_equal = P.NotEqual() self.num_cls = num_cls self.ignore_label = ignore_label self.mul = P.Mul() self.sum = P.ReduceSum(False) self.div = P.RealDiv() self.transpose = P.Transpose() self.reshape = P.Reshape()
def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0): super(Adam, self).__init__(learning_rate, params) _check_param_value(beta1, beta2, eps, weight_decay) validator.check_type("use_locking", use_locking, [bool]) validator.check_type("use_nesterov", use_nesterov, [bool]) validator.check_type("loss_scale", loss_scale, [float]) validator.check_number_range("loss_scale", loss_scale, 1.0, float("inf"), Rel.INC_LEFT) self.dynamic_lr = False if isinstance(learning_rate, Iterable) or \ (isinstance(learning_rate, Tensor) and learning_rate.dim() == 1): self.dynamic_lr = True self.gather = P.GatherV2() self.assignadd = P.AssignAdd() self.global_step = Parameter(initializer(0, [1], mstype.int32), name="global_step") self.axis = 0 self.beta1 = Tensor(beta1, mstype.float32) self.beta2 = Tensor(beta2, mstype.float32) self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power") self.beta2_power = Parameter(initializer(1, [1], mstype.float32), name="beta2_power") self.eps = eps self.moment1 = self.parameters.clone(prefix="moment1", init='zeros') self.moment2 = self.parameters.clone(prefix="moment2", init='zeros') self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) self.weight_decay = weight_decay * loss_scale self.reciprocal_scale = 1.0 / loss_scale self.pow = P.Pow() self.sqrt = P.Sqrt() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.realdiv = P.RealDiv()
def construct(self, a, b, x): if a < b: a = P.TensorAdd()(a, b) else: a = P.Sub()(a, b) if a == x: a = P.Mul()(a, b) else: a = P.RealDiv()(a, b) if b == x: b = P.TensorAdd()(a, b) else: b = P.TensorAdd()(a, x) a = a * b out = a + b + x return out
def __init__(self, input_dim, output_dim, weight_bias_init, act_str, keep_prob=0.7, scale_coef=1.0, convert_dtype=True): super(DenseLayer, self).__init__() weight_init, bias_init = weight_bias_init self.weight = init_method( weight_init, [input_dim, output_dim], name="weight") self.bias = init_method(bias_init, [output_dim], name="bias") self.act_func = self._init_activation(act_str) self.matmul = P.MatMul(transpose_b=False) self.bias_add = P.BiasAdd() self.cast = P.Cast() #self.dropout = Dropout(keep_prob=keep_prob) self.mul = P.Mul() self.realDiv = P.RealDiv() self.scale_coef = scale_coef self.convert_dtype = convert_dtype
def __init__(self, sparse=False): super(SoftmaxCrossEntropyExpand, self).__init__() self.exp = P.Exp() self.sum = P.ReduceSum(keep_dims=True) self.onehot = P.OneHot() self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.div = P.RealDiv() self.log = P.Log() self.sum_cross_entropy = P.ReduceSum(keep_dims=False) self.mul = P.Mul() self.mul2 = P.Mul() self.mean = P.ReduceMean(keep_dims=False) self.sparse = sparse self.max = P.ReduceMax(keep_dims=True) self.sub = P.Sub() self.eps = Tensor(1e-24, mstype.float32)
def test_nobroadcast(): context.set_context(mode=context.GRAPH_MODE, device_target='GPU') np.random.seed(42) x1_np = np.random.rand(10, 20).astype(np.float32) x2_np = np.random.rand(10, 20).astype(np.float32) x1_np_int32 = np.random.randint(0, 100, (10, 20)).astype(np.int32) x2_np_int32 = np.random.randint(0, 100, (10, 20)).astype(np.int32) output_ms = P.Minimum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.minimum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Maximum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.maximum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Greater()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np > x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Greater()(Tensor(x1_np_int32), Tensor(x2_np_int32)) output_np = x1_np_int32 > x2_np_int32 assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Less()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np < x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Less()(Tensor(x1_np_int32), Tensor(x2_np_int32)) output_np = x1_np_int32 < x2_np_int32 assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Pow()(Tensor(x1_np), Tensor(x2_np)) output_np = np.power(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.RealDiv()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np / x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Mul()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np * x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Sub()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np - x2_np assert np.allclose(output_ms.asnumpy(), output_np)
def test_nobroadcast_fp16(): context.set_context(mode=context.GRAPH_MODE, device_target='GPU') np.random.seed(42) x1_np = np.random.rand(10, 20).astype(np.float16) x2_np = np.random.rand(10, 20).astype(np.float16) output_ms = P.Minimum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.minimum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Maximum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.maximum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Greater()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np > x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Less()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np < x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Pow()(Tensor(x1_np), Tensor(x2_np)) output_np = np.power(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.RealDiv()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np / x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Mul()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np * x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Sub()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np - x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.DivNoNan()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np / x2_np assert np.allclose(output_ms.asnumpy(), output_np) x2_np_zero = np.zeros_like(x2_np) output_ms = P.DivNoNan()(Tensor(x1_np), Tensor(x2_np_zero)) assert np.allclose(output_ms.asnumpy(), x2_np_zero)
def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(Adam, self).__init__(learning_rate, params, weight_decay, loss_scale, decay_filter) _check_param_value(beta1, beta2, eps, weight_decay, self.cls_name) validator.check_value_type("use_locking", use_locking, [bool], self.cls_name) validator.check_value_type("use_nesterov", use_nesterov, [bool], self.cls_name) validator.check_value_type("loss_scale", loss_scale, [float], self.cls_name) validator.check_number_range("loss_scale", loss_scale, 1.0, float("inf"), Rel.INC_LEFT, self.cls_name) self.beta1 = Tensor(beta1, mstype.float32) self.beta2 = Tensor(beta2, mstype.float32) self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power") self.beta2_power = Parameter(initializer(1, [1], mstype.float32), name="beta2_power") self.eps = eps self.moment1 = self.parameters.clone(prefix="moment1", init='zeros') self.moment2 = self.parameters.clone(prefix="moment2", init='zeros') self.decay_tf = tuple(decay_filter(x) for x in self.parameters) self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) self.pow = P.Pow() self.sqrt = P.Sqrt() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.realdiv = P.RealDiv()
def __init__(self, probs=None, logits=None, seed=None, dtype=mstype.int32, name="Categorical"): param = dict(locals()) valid_dtype = mstype.int_type check_type(dtype, valid_dtype, "Categorical") super(Categorical, self).__init__(seed, dtype, name, param) if (probs is None) == (logits is None): raise_probs_logits_error() self.reduce_sum = P.ReduceSum(keep_dims=True) self.reduce_sum1 = P.ReduceSum(keep_dims=False) self.log = P.Log() self.exp = P.Exp() self.shape = P.Shape() self.reshape = P.Reshape() self.div = P.RealDiv() self.size = P.Size() self.mutinomial = P.Multinomial(seed=self.seed) self.cast = P.Cast() self.expandim = P.ExpandDims() self.gather = P.GatherNd() self.concat = P.Concat(-1) self.transpose = P.Transpose() if probs is not None: self._probs = cast_to_tensor(probs, mstype.float32) input_sum = self.reduce_sum(self._probs, -1) self._probs = self.div(self._probs, input_sum) self._logits = probs_to_logits(self._probs) self._param = self._probs else: self._logits = cast_to_tensor(logits, mstype.float32) input_sum = self.reduce_sum(self.exp(self._logits), -1) self._logits = self._logits - self.log(input_sum) self._probs = logits_to_probs(self._logits) self._param = self._logits self._num_events = self.shape(self._param)[-1] self._param2d = self.reshape(self._param, (-1, self._num_events)) self._batch_shape = self.shape(self._param)[:-1] self._batch_shape_n = (1, ) * len(self._batch_shape)
def construct(self, a, b, x): add = P.TensorAdd() sub = P.Sub() mul = P.Mul() div = P.RealDiv() if 2 < 12: a = add(a, b) else: a = sub(a, b) if 2 > 1: a = mul(a, b) else: a = div(a, b) if 2 == 1: b = add(a, b) else: b = add(a, x) a = a * b out = a + b + x return out
def construct(self, a, b, x): add = P.Add() sub = P.Sub() mul = P.Mul() div = P.RealDiv() if a < b: a = add(a, b) else: a = sub(a, b) if 2 > 1: a = mul(a, b) else: a = div(a, b) if b == x: b = add(a, b) else: b = add(a, x) a = a * b out = a + b + x return out
def __init__(self, vocab_size, embedding_dims, num_class): super(FastText, self).__init__() self.vocab_size = vocab_size self.embeding_dims = embedding_dims self.num_class = num_class self.embeding_func = nn.Embedding(vocab_size=self.vocab_size, embedding_size=self.embeding_dims, padding_idx=0, embedding_table='Zeros') self.fc = nn.Dense(self.embeding_dims, out_channels=self.num_class, weight_init=XavierUniform(1)).to_float( mstype.float16) self.reducesum = P.ReduceSum() self.expand_dims = P.ExpandDims() self.squeeze = P.Squeeze(axis=1) self.cast = P.Cast() self.tile = P.Tile() self.realdiv = P.RealDiv() self.fill = P.Fill() self.log_softmax = nn.LogSoftmax(axis=1)
def __init__(self, probs=None, logits=None, seed=0, dtype=mstype.int32, name="Categorical"): param = dict(locals()) super(Categorical, self).__init__(seed, dtype, name, param) if (probs is None) == (logits is None): raise ValueError( "Either 'prob' or 'logits' must be specified, but not both.") self.reduce_sum = P.ReduceSum(keep_dims=True) self.log = P.Log() self.exp = P.Exp() self.shape = P.Shape() self.reshape = P.Reshape() self.div = P.RealDiv() self.size = P.Size() self.mutinomial = P.Multinomial(seed=seed) self.cast = P.Cast() self.expandim = P.ExpandDims() self.gather = P.GatherNd() self.concat = P.Concat(-1) if probs is not None: self._probs = cast_to_tensor(probs, mstype.float32) input_sum = self.reduce_sum(self._probs, -1) self._probs = self.div(self._probs, input_sum) self._logits = probs_to_logits(self._probs) self._param = self._probs else: self._logits = cast_to_tensor(logits, mstype.float32) input_sum = self.reduce_sum(self.exp(self._logits), -1) self._logits = self._logits - self.log(input_sum) self._probs = logits_to_probs(self._logits) self._param = self._logits self._num_events = self.shape(self._param)[-1] self._param2d = self.reshape(self._param, (-1, self._num_events)) self._batch_shape = self.shape(self._param2d)[0]
def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0): super(Adam, self).__init__(learning_rate, params, weight_decay, loss_scale) _check_param_value(beta1, beta2, eps, weight_decay, self.cls_name) validator.check_value_type("use_locking", use_locking, [bool], self.cls_name) validator.check_value_type("use_nesterov", use_nesterov, [bool], self.cls_name) validator.check_value_type("loss_scale", loss_scale, [float], self.cls_name) self.beta1 = Tensor(beta1, mstype.float32) self.beta2 = Tensor(beta2, mstype.float32) self.beta1_power = Parameter(initializer(1, [1], mstype.float32)) self.beta2_power = Parameter(initializer(1, [1], mstype.float32)) self.eps = eps self.moment1 = self.parameters.clone(prefix="moment1", init='zeros') self.moment2 = self.parameters.clone(prefix="moment2", init='zeros') self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) self.pow = P.Pow() self.sqrt = P.Sqrt() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.realdiv = P.RealDiv() self.lr_scalar = P.ScalarSummary()
def test_broadcast_fp16(): context.set_context(mode=context.GRAPH_MODE, device_target='GPU') x1_np = np.random.rand(3, 1, 5, 1).astype(np.float16) x2_np = np.random.rand(1, 4, 1, 6).astype(np.float16) output_ms = P.Minimum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.minimum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Maximum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.maximum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Greater()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np > x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Less()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np < x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Pow()(Tensor(x1_np), Tensor(x2_np)) output_np = np.power(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.RealDiv()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np / x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Mul()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np * x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Sub()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np - x2_np assert np.allclose(output_ms.asnumpy(), output_np)
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ from mindspore.ops import Primitive from mindspore.ops import operations as P add = P.TensorAdd() mul = P.Mul() real_div = P.RealDiv() rsqrt = P.Rsqrt() sqrt = P.Sqrt() make_tuple = Primitive('make_tuple') tuple_getitem = Primitive('tuple_getitem') LambNextMVWithDecayV1 = Primitive('LambNextMVWithDecayV1') class FnDict: def __init__(self): self.fnDict = {} def __call__(self, fn): self.fnDict[fn.__name__] = fn def __getitem__(self, name):
def __init__(self, batch_size, seq_length, vocab_size, decoder, beam_width=4, length_penalty_weight=1.0, max_decode_length=128, sos_id=1, eos_id=2, compute_type=mstype.float32): super(BeamSearchDecoder, self).__init__(auto_prefix=False) self.seq_length = seq_length self.batch_size = batch_size self.vocab_size = vocab_size self.beam_width = beam_width self.length_penalty_weight = length_penalty_weight self.max_decode_length = max_decode_length self.decoder = decoder self.add = P.TensorAdd() self.expand = P.ExpandDims() self.reshape = P.Reshape() self.shape_flat = (-1, ) self.shape = P.Shape() self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]), mstype.float32) self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF), mstype.float32) self.select = P.Select() self.flat_shape = (batch_size, beam_width * vocab_size) self.topk = P.TopK(sorted=True) self.floor_div = P.FloorDiv() self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32) self.real_div = P.RealDiv() self.mod = Mod() self.equal = P.Equal() self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id), mstype.int32) beam_ids = np.tile( np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1]) self.beam_ids = Tensor(beam_ids, mstype.int32) batch_ids = np.arange(batch_size * beam_width).reshape( (batch_size, beam_width)) // beam_width self.batch_ids = Tensor(batch_ids, mstype.int32) self.concat = P.Concat(axis=-1) self.gather_nd = P.GatherNd() self.greater_equal = P.GreaterEqual() self.sub = P.Sub() self.cast = P.Cast() self.zeroslike = P.ZerosLike() # init inputs and states self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id), mstype.int32) self.init_seq = Tensor(np.full([batch_size, beam_width, 1], sos_id), mstype.int32) init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]), [batch_size, 1]) self.init_scores = Tensor(init_scores, mstype.float32) self.init_finished = Tensor( np.zeros([batch_size, beam_width], dtype=np.bool)) self.init_length = Tensor( np.zeros([batch_size, beam_width], dtype=np.int32)) self.length_penalty = LengthPenalty(weight=length_penalty_weight) self.one = Tensor(1, mstype.int32)
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ from mindspore.ops import Primitive from mindspore.ops import operations as P from mindspore.ops import _constants as Constants Sub = P.Sub() Mul = P.Mul() RealDiv = P.RealDiv() Select = P.Select() Greater = P.Greater() make_tuple = Primitive('MakeTuple') tuple_getitem = Primitive(Constants.kTupleGetItem) LambUpdateWithLrV2 = Primitive('LambUpdateWithLrV2') class FnDict: def __init__(self): self.fnDict = {} def __call__(self, fn): self.fnDict[fn.__name__] = fn def __getitem__(self, name):
def __init__(self): super(NetRealDiv, self).__init__() self.divide = P.RealDiv()
def construct(self, logits, label): label = self.one_hot(label, F.shape(logits)[1], self.on_value, self.off_value) loss = self.cross_entropy(logits, label)[0] loss = P.RealDiv()(P.ReduceSum()(loss, -1), self.num) return loss
def _grad_div(val, grad): div = P.RealDiv() mul = P.Mul() grad = mul(grad, 10.0) ret = div(grad, val) return ret
def test_broadcast_diff_dims(): context.set_context(mode=context.GRAPH_MODE, device_target='GPU') np.random.seed(42) x1_np = np.random.rand(2).astype(np.float32) x2_np = np.random.rand(2, 1).astype(np.float32) x1_np_int32 = np.random.randint(0, 100, (2)).astype(np.int32) x2_np_int32 = np.random.randint(0, 100, (2, 1)).astype(np.int32) output_ms = P.Minimum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.minimum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Maximum()(Tensor(x1_np), Tensor(x2_np)) output_np = np.maximum(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Greater()(Tensor(x1_np_int32), Tensor(x2_np_int32)) output_np = x1_np_int32 > x2_np_int32 assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Greater()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np > x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Less()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np < x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Less()(Tensor(x1_np_int32), Tensor(x2_np_int32)) output_np = x1_np_int32 < x2_np_int32 assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Pow()(Tensor(x1_np), Tensor(x2_np)) output_np = np.power(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.RealDiv()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np / x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Mul()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np * x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.Sub()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np - x2_np assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.DivNoNan()(Tensor(x1_np), Tensor(x2_np)) output_np = x1_np / x2_np assert np.allclose(output_ms.asnumpy(), output_np) x2_np_zero = np.zeros_like(x2_np) output_ms = P.DivNoNan()(Tensor(x1_np), Tensor(x2_np_zero)) assert np.allclose(output_ms.asnumpy(), x2_np_zero) output_ms = P.Mod()(Tensor(x1_np), Tensor(x2_np)) output_np = np.fmod(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np) output_ms = P.FloorMod()(Tensor(x1_np), Tensor(x2_np)) output_np = np.mod(x1_np, x2_np) assert np.allclose(output_ms.asnumpy(), output_np)
def __init__(self): super(Net, self).__init__() self.realdiv = P.RealDiv()
'block': P.RandomChoiceWithMask(256), 'desc_inputs': [Tensor(np.random.rand(24000, 4).astype(np.bool_))], 'desc_bprop': [[256,4], [256,4]], 'skip': ['backward']}), ('LessEqual', { 'block': P.LessEqual(), 'desc_inputs': [Tensor(np.random.rand(4).astype(np.float16)), Tensor(np.random.rand(4).astype(np.float16))], 'skip': ['backward']}), ('Less', { 'block': P.Less(), 'desc_inputs': [[2, 1, 4, 5], [2, 1, 4, 5]], 'desc_bprop': [Tensor(np.zeros((2, 1, 4, 5), np.bool_))], 'skip': ['backward']}), ('RealDiv_0', { 'block': P.RealDiv(), 'desc_const': [Tensor(2048.0), Tensor(0.0)], 'desc_inputs': [], 'skip': ['backward']}), ('RealDiv', { 'block': P.RealDiv(), 'desc_inputs': [[4], Tensor(np.ones(4).astype(np.float32))], 'desc_bprop': [[4]]}), ('RealDiv_1', { 'block': P.RealDiv(), 'desc_inputs': [[512, 1024], [512, 1024]], 'desc_bprop': [[512, 1024]]}), ('FloorDiv', { 'block': P.FloorDiv(), 'desc_inputs': [Tensor(np.random.rand(4).astype(np.float16)), Tensor(np.random.rand(4).astype(np.float16))],
def __init__(self): super().__init__() self.add = P.TensorAdd() self.sub = P.Sub() self.mul = P.Mul() self.div = P.RealDiv()
# input two tensors, their shapes do not match ('Maximum2', { 'block': (P.Maximum(), { 'exception': ValueError, 'error_keywords': ['Maximum'] }), 'desc_inputs': [ Tensor(np.ones([3, 5]).astype(np.float32)), Tensor(np.ones([3, 4]).astype(np.float32)) ], 'skip': ['backward'] }), # input two tensors, their shapes do not match ('RealDiv2', { 'block': (P.RealDiv(), { 'exception': ValueError, 'error_keywords': ['RealDiv'] }), 'desc_inputs': [ Tensor(np.ones([3, 5]).astype(np.float32)), Tensor(np.ones([3, 4]).astype(np.float32)) ], 'skip': ['backward'] }), # input two tensors, their shapes do not match ('Div2', { 'block': (P.Div(), { 'exception': ValueError, 'error_keywords': ['Div']
'desc_inputs': [5.0, Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # input two tensors, but element types are not same ('Maximum1', { 'block': (P.Maximum(), {'exception': TypeError, 'error_keywords': ['Maximum']}), 'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # input two tensors, their shapes do not match ('Maximum2', { 'block': (P.Maximum(), {'exception': ValueError, 'error_keywords': ['Maximum']}), 'desc_inputs': [Tensor(np.ones([3, 5]).astype(np.float32)), Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # one input is scalar, and another is Tensor(float32) ('RealDiv0', { 'block': (P.RealDiv(), {'exception': TypeError, 'error_keywords': ['RealDiv']}), 'desc_inputs': [5.0, Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # input two tensors, but element types are not same ('RealDiv1', { 'block': (P.RealDiv(), {'exception': TypeError, 'error_keywords': ['RealDiv']}), 'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # input two tensors, their shapes do not match ('RealDiv2', { 'block': (P.RealDiv(), {'exception': ValueError, 'error_keywords': ['RealDiv']}), 'desc_inputs': [Tensor(np.ones([3, 5]).astype(np.float32)), Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # one input is scalar, and another is Tensor(float32) ('Div0', {
def __init__(self, batch_size, seq_length, vocab_size, decoder, beam_width=4, decoder_layers_nums=4, length_penalty_weight=0.6, cov_penalty_factor=0.1, hidden_size=1024, max_decode_length=64, sos_id=2, eos_id=3, compute_type=mstype.float32): super(BeamSearchDecoder, self).__init__() self.encoder_length = seq_length self.hidden_size = hidden_size self.batch_size = batch_size self.vocab_size = vocab_size self.beam_width = beam_width self.decoder_layers_nums = decoder_layers_nums self.length_penalty_weight = length_penalty_weight self.cov_penalty_factor = cov_penalty_factor self.max_decode_length = max_decode_length self.decoder = decoder self.add = P.TensorAdd() self.expand = P.ExpandDims() self.reshape = P.Reshape() self.shape_flat = (-1,) self.shape = P.Shape() self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]), mstype.float32) self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF), mstype.float32) self.select = P.Select() self.flat_shape = (batch_size, beam_width * vocab_size) self.topk = P.TopK(sorted=True) self.floor_div = P.FloorDiv() self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32) self.real_div = P.RealDiv() self.mod = Mod() self.equal = P.Equal() self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id), mstype.int32) beam_ids = np.tile(np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1]) self.beam_ids = Tensor(beam_ids, mstype.int32) batch_ids = np.arange(batch_size * beam_width).reshape((batch_size, beam_width)) // beam_width self.batch_ids = Tensor(batch_ids, mstype.int32) self.concat = P.Concat(axis=-1) self.gather_nd = P.GatherNd() self.start = Tensor(0, dtype=mstype.int32) self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id), mstype.int32) self.init_seq = Tensor(np.full([batch_size, beam_width, self.max_decode_length], sos_id), mstype.int32) init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]), [batch_size, 1]) self.init_scores = Tensor(init_scores, mstype.float32) self.init_finished = Tensor(np.zeros([batch_size, beam_width], dtype=np.bool)) self.init_length = Tensor(np.zeros([batch_size, beam_width], dtype=np.int32)) self.length_penalty = LengthPenalty(weight=length_penalty_weight) self.one = Tensor(1, mstype.int32) self.prob_concat = P.Concat(axis=1) self.cast = P.Cast() self.decoder_hidden_state = Tensor(np.zeros([self.decoder_layers_nums, 2, self.batch_size * self.beam_width, hidden_size]), mstype.float32) self.zeros_scores = Tensor(np.zeros([batch_size, beam_width], dtype=np.float)) self.active_index = Tensor(np.ones([batch_size, beam_width], dtype=np.int32)) self.init_zeros = Tensor(np.zeros([batch_size, beam_width], dtype=np.int32)) self.init_ones = Tensor(np.ones([batch_size, beam_width], dtype=np.float32)) self.accu_attn_scores = Tensor(np.zeros([batch_size, beam_width, self.encoder_length], dtype=np.float32)) self.zeros = Tensor([0], mstype.int32) self.eos_tensor = Tensor(np.full([batch_size, beam_width, beam_width], eos_id), mstype.int32) self.ones_3d = Tensor(np.full([batch_size, beam_width, self.encoder_length], 1), mstype.float32) self.neg_inf_3d = Tensor(np.full([batch_size, beam_width, self.encoder_length], -INF), mstype.float32) self.zeros_3d = Tensor(np.full([batch_size, beam_width, self.encoder_length], 0), mstype.float32) self.zeros_2d = Tensor(np.full([batch_size * beam_width, self.encoder_length], 0), mstype.int32) self.argmin = P.ArgMinWithValue(axis=1) self.reducesum = P.ReduceSum() self.div = P.Div() self.shape_op = P.Shape() self.mul = P.Mul() self.log = P.Log() self.less = P.Less() self.tile = P.Tile() self.noteq = P.Neg() self.zeroslike = P.ZerosLike() self.greater_equal = P.GreaterEqual() self.sub = P.Sub()
def __init__(self, args, strategy): super(SemiAutoOneHotNet, self).__init__() self.a = args.a self.b = args.b self.c = args.c self.d = args.d self.e = args.e self.cast = P.Cast() self.cast.set_strategy(strategy=strategy.twod_strategy) self.cast1 = P.Cast() self.cast1.set_strategy(strategy=strategy.twod_strategy) self.cast2 = P.Cast() self.cast2.set_strategy(strategy=strategy.twod_strategy) self.cast3 = P.Cast() self.cast3.set_strategy(strategy=strategy.scalar_strategy) self.cast4 = P.Cast() self.cast4.set_strategy(strategy=strategy.scalar_strategy) self.a_const = Tensor(self.a, dtype=mstype.float32) self.b_const = Tensor(self.b, dtype=mstype.float32) self.c_const = Tensor(self.c, dtype=mstype.float32) self.d_const = Tensor(self.d, dtype=mstype.float32) self.e_const = Tensor(self.e, dtype=mstype.float32) self.m_const_zero = Tensor(0, dtype=mstype.float32) self.a_const_one = Tensor(1, dtype=mstype.float32) self.onehot = P.OneHot() self.onehot.set_strategy(strategy=strategy.onehot_strategy) self.exp = P.Exp() self.exp.set_strategy(strategy=strategy.twod_strategy) self.exp2 = P.Exp() self.exp2.set_strategy(strategy=strategy.twod_strategy) self.exp3 = P.Exp() self.exp3.set_strategy(strategy=strategy.twod_strategy) self.mul_const = P.Mul() self.mul_const.set_strategy(strategy=strategy.scalar_twod_strategy) self.mul_const2 = P.TensorAdd() self.mul_const2.set_strategy(strategy=strategy.scalar_twod_strategy) self.mul_const3 = P.Sub() self.mul_const3.set_strategy(strategy=strategy.twod_scalar_strategy) self.mul_const4 = P.Sub() self.mul_const4.set_strategy(strategy=strategy.scalar_twod_strategy) self.mul_const5 = P.Mul() self.mul_const5.set_strategy(strategy=strategy.twod_scalar_strategy) self.mul = P.Mul() self.mul.set_strategy(strategy=strategy.twod_twod_strategy) self.mul2 = P.Mul() self.mul2.set_strategy(strategy=strategy.twod_twod_strategy) self.mul3 = P.TensorAdd() self.mul3.set_strategy(strategy=strategy.twod_twod_strategy) self.mul4 = P.Sub() self.mul4.set_strategy(strategy=strategy.twod_twodbc_strategy) self.mul5 = P.RealDiv() self.mul5.set_strategy(strategy=strategy.twod_twodbc_strategy) self.mul6 = P.Mul() self.mul6.set_strategy(strategy=strategy.twod_twod_strategy) self.mul7 = P.Mul() self.mul7.set_strategy(strategy=strategy.twod_scalar_strategy) self.mul8 = P.RealDiv() self.mul8.set_strategy(strategy=strategy.scalar_scalar_strategy) self.mul9 = P.TensorAdd() self.mul9.set_strategy(strategy=strategy.twod_scalar_strategy) self.reduce_max = P.ReduceMax(keep_dims=True) self.reduce_max.set_strategy(strategy=strategy.twod_strategy) self.reduce_sum = P.ReduceSum(keep_dims=False) self.reduce_sum.set_strategy(strategy=strategy.twod_strategy) self.reduce_sum_2 = P.ReduceSum(keep_dims=False) self.reduce_sum_2.set_strategy(strategy=strategy.twod_strategy) self.reduce_sum_3 = P.ReduceSum(keep_dims=False) self.reduce_sum_3.set_strategy(strategy=strategy.oned_strategy) self.reshape = P.Reshape() self.log = P.Log() self.log.set_strategy(strategy=strategy.twod_strategy) self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.normalize = P.L2Normalize(axis=1) self.normalize.set_strategy(strategy=strategy.twod_strategy_m) self.normalize2 = P.L2Normalize(axis=1) self.normalize2.set_strategy(strategy=strategy.twod_strategy_m) self.fc = P.MatMul(transpose_b=True) self.fc.set_strategy(strategy=strategy.twodbc_twod_strategy) weight_shape = [args.num_classes, args.emb_size] weight_np = np.zeros(weight_shape, np.float32) self.weight = Parameter(Tensor(weight_np), name='model_parallel_weight')