def __init__(self, strategy0=None, strategy1=None): super(AddRelu, self).__init__() self.add = P.TensorAdd().set_strategy(strategy=strategy0) self.relu = P.ReLU().set_strategy(strategy=strategy1)
def __init__(self, strategy1, strategy2): super().__init__() self.matmul = P.MatMul().set_strategy(strategy1) self.add = P.TensorAdd().set_strategy(strategy2) self.b = Tensor(0.9, ms.float32)
def __init__(self): super().__init__() self.op = P.TensorAdd()
def __init__(self, batch_size, from_tensor_width, to_tensor_width, from_seq_length, to_seq_length, num_attention_heads=1, size_per_head=512, query_act=None, key_act=None, value_act=None, has_attention_mask=False, attention_probs_dropout_prob=0.0, use_one_hot_embeddings=False, initializer_range=0.02, do_return_2d_tensor=False, use_relative_positions=False, compute_type=mstype.float32): super(BertAttention, self).__init__() self.batch_size = batch_size self.from_seq_length = from_seq_length self.to_seq_length = to_seq_length self.num_attention_heads = num_attention_heads self.size_per_head = size_per_head self.has_attention_mask = has_attention_mask self.use_relative_positions = use_relative_positions self.scores_mul = 1.0 / math.sqrt(float(self.size_per_head)) self.reshape = P.Reshape() self.shape_from_2d = (-1, from_tensor_width) self.shape_to_2d = (-1, to_tensor_width) weight = TruncatedNormal(initializer_range) units = num_attention_heads * size_per_head self.query_layer = Dense_Thor(in_channels=from_tensor_width, out_channels=units, weight_init=weight, has_bias=True, bias_init='zeros', damping=damping, loss_scale=loss_scale, frequency=frequency, activation=query_act, batch_size=batch_size).to_float(compute_type) self.key_layer = Dense_Thor(in_channels=to_tensor_width, out_channels=units, weight_init=weight, has_bias=True, bias_init='zeros', damping=damping, loss_scale=loss_scale, frequency=frequency, activation=key_act, batch_size=batch_size).to_float(compute_type) self.value_layer = Dense_Thor(in_channels=to_tensor_width, out_channels=units, weight_init=weight, has_bias=True, bias_init='zeros', damping=damping, loss_scale=loss_scale, frequency=frequency, activation=value_act, batch_size=batch_size).to_float(compute_type) self.shape_from = (batch_size, from_seq_length, num_attention_heads, size_per_head) self.shape_to = ( batch_size, to_seq_length, num_attention_heads, size_per_head) self.matmul_trans_b = P.BatchMatMul(transpose_b=True) self.multiply = P.Mul() self.transpose = P.Transpose() self.trans_shape = (0, 2, 1, 3) self.trans_shape_relative = (2, 0, 1, 3) self.trans_shape_position = (1, 2, 0, 3) self.multiply_data = -10000.0 self.batch_num = batch_size * num_attention_heads self.matmul = P.BatchMatMul() self.softmax = nn.Softmax() self.dropout = nn.Dropout(1 - attention_probs_dropout_prob) if self.has_attention_mask: self.expand_dims = P.ExpandDims() self.sub = P.Sub() self.add = P.TensorAdd() self.cast = P.Cast() self.get_dtype = P.DType() if do_return_2d_tensor: self.shape_return = (batch_size * from_seq_length, num_attention_heads * size_per_head) else: self.shape_return = (batch_size, from_seq_length, num_attention_heads * size_per_head) self.cast_compute_type = SaturateCast(dst_type=compute_type) if self.use_relative_positions: self._generate_relative_positions_embeddings = \ RelaPosEmbeddingsGenerator(length=to_seq_length, depth=size_per_head, max_relative_position=16, initializer_range=initializer_range, use_one_hot_embeddings=use_one_hot_embeddings)
def __init__(self, strategy1, strategy2, strategy3): super().__init__() self.mul = P.Mul().set_strategy(strategy1) self.reduce_max = P.ReduceMax(keep_dims=False).set_strategy(strategy2) self.add = P.TensorAdd().set_strategy(strategy3)
def __init__(self, batch_size, seq_length, vocab_size, decoder, beam_width=4, decoder_layers_nums=4, length_penalty_weight=0.6, cov_penalty_factor=0.1, hidden_size=1024, max_decode_length=64, sos_id=2, eos_id=3, is_using_while=True, compute_type=mstype.float32): super(BeamSearchDecoder, self).__init__() self.encoder_length = seq_length self.hidden_size = hidden_size self.batch_size = batch_size self.vocab_size = vocab_size self.beam_width = beam_width self.decoder_layers_nums = decoder_layers_nums self.length_penalty_weight = length_penalty_weight self.cov_penalty_factor = cov_penalty_factor self.max_decode_length = max_decode_length self.decoder = decoder self.is_using_while = is_using_while self.add = P.TensorAdd() self.expand = P.ExpandDims() self.reshape = P.Reshape() self.shape_flat = (-1, ) self.shape = P.Shape() self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]), mstype.float32) self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF), mstype.float32) self.select = P.Select() self.flat_shape = (batch_size, beam_width * vocab_size) self.topk = P.TopK(sorted=True) self.floor_div = P.FloorDiv() self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32) self.real_div = P.RealDiv() self.mod = Mod() self.equal = P.Equal() self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id), mstype.int32) beam_ids = np.tile( np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1]) self.beam_ids = Tensor(beam_ids, mstype.int32) batch_ids = np.arange(batch_size * beam_width).reshape( (batch_size, beam_width)) // beam_width self.batch_ids = Tensor(batch_ids, mstype.int32) self.concat = P.Concat(axis=-1) self.gather_nd = P.GatherNd() self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id), mstype.int32) if self.is_using_while: self.start = Tensor(0, dtype=mstype.int32) self.init_seq = Tensor( np.full([batch_size, beam_width, self.max_decode_length], sos_id), mstype.int32) else: self.init_seq = Tensor( np.full([batch_size, beam_width, 1], sos_id), mstype.int32) init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]), [batch_size, 1]) self.init_scores = Tensor(init_scores, mstype.float32) self.init_finished = Tensor( np.zeros([batch_size, beam_width], dtype=np.bool)) self.init_length = Tensor( np.zeros([batch_size, beam_width], dtype=np.int32)) self.length_penalty = LengthPenalty(weight=length_penalty_weight) self.one = Tensor(1, mstype.int32) self.prob_concat = P.Concat(axis=1) self.cast = P.Cast() self.decoder_hidden_state = Tensor( np.zeros([ self.decoder_layers_nums, 2, self.batch_size * self.beam_width, hidden_size ]), mstype.float32) self.zeros_scores = Tensor( np.zeros([batch_size, beam_width], dtype=np.float)) self.active_index = Tensor( np.ones([batch_size, beam_width], dtype=np.int32)) self.init_zeros = Tensor( np.zeros([batch_size, beam_width], dtype=np.int32)) self.init_ones = Tensor( np.ones([batch_size, beam_width], dtype=np.float32)) self.accu_attn_scores = Tensor( np.zeros([batch_size, beam_width, self.encoder_length], dtype=np.float32)) self.zeros = Tensor([0], mstype.int32) self.eos_tensor = Tensor( np.full([batch_size, beam_width, beam_width], eos_id), mstype.int32) self.ones_3d = Tensor( np.full([batch_size, beam_width, self.encoder_length], 1), mstype.float32) self.neg_inf_3d = Tensor( np.full([batch_size, beam_width, self.encoder_length], -INF), mstype.float32) self.zeros_3d = Tensor( np.full([batch_size, beam_width, self.encoder_length], 0), mstype.float32) self.zeros_2d = Tensor( np.full([batch_size * beam_width, self.encoder_length], 0), mstype.int32) self.argmin = P.ArgMinWithValue(axis=1) self.reducesum = P.ReduceSum() self.div = P.Div() self.shape_op = P.Shape() self.mul = P.Mul() self.log = P.Log() self.less = P.Less() self.tile = P.Tile() self.noteq = P.Neg() self.zeroslike = P.ZerosLike() self.greater_equal = P.GreaterEqual() self.sub = P.Sub()
def __init__(self): super(MathBinaryNet1, self).__init__() self.add = P.TensorAdd() self.mul = P.Mul() self.max = P.Maximum() self.number = 3
def __init__(self): super(Net1, self).__init__() self.add = P.TensorAdd()
def construct(self, x, y): add = P.TensorAdd() result = add(x, y) return result
return self.add(x, y) test_case_math_ops = [ ('Neg', { 'block': P.Neg(), 'desc_inputs': [[1, 3, 4, 4]], 'desc_bprop': [[1, 3, 4, 4]] }), ('Sub', { 'block': P.Sub(), 'desc_inputs': [[3, 5], [2, 3, 3, 5]], 'desc_bprop': [[2, 3, 3, 5]] }), ('TensorAdd', { 'block': P.TensorAdd(), 'desc_inputs': [[3, 5], [2, 3, 3, 5]], 'desc_bprop': [[2, 3, 3, 5]] }), ('Mul0', { 'block': P.Mul(), 'desc_inputs': [[2, 3, 3, 5], [2, 3, 3, 5]], 'desc_bprop': [[2, 3, 3, 5]] }), ('Mul1', { 'block': P.Mul(), 'desc_inputs': [[2, 3, 1, 1], [2, 3, 3, 5]], 'desc_bprop': [[2, 3, 3, 5]] }), ('Mul2', { 'block': P.Mul(),
def __init__(self, ): super(SummaryNet, self).__init__() self.s = P.ScalarSummary() self.add = P.TensorAdd()
def __init__(self, in_channels, out_channels, weight_init='normal', bias_init='zeros', has_bias=True, activation=None): super(Dense_Thor, self).__init__() self.thor = True self.in_channels = Validator.check_positive_int(in_channels) self.out_channels = Validator.check_positive_int(out_channels) self.has_bias = Validator.check_bool(has_bias) if isinstance(weight_init, Tensor): if weight_init.dim() != 2 or weight_init.shape[0] != out_channels or \ weight_init.shape[1] != in_channels: raise ValueError("Weight init shape error.") self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") self.bias = None if self.has_bias: if isinstance(bias_init, Tensor): if bias_init.dim() != 1 or bias_init.shape[0] != out_channels: raise ValueError("Bias init shape error.") self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") self.bias_add = P.BiasAdd() self.matmul = P.MatMul(transpose_b=True) self.activation = get_activation(activation) self.activation_flag = self.activation is not None self.matrix_A = Parameter(Tensor( np.zeros([in_channels, in_channels]).astype(np.float32)), name='matrix_A', requires_grad=False) self.shape = P.Shape() self.reshape = P.Reshape() self.transpose = P.Transpose() self.mul = P.Mul() self.is_Ascend = True if context.get_context("device_target") == "Ascend": if out_channels == 1001: self.matrix_G = Parameter(Tensor( np.zeros([1024, 1024]).astype(np.float32)), name='matrix_G', requires_grad=False) self.pad = P.Pad(((0, 23), (0, 23))) self.pad1 = P.Pad(((0, 7), (0, 7))) self.slice = P.Slice() self.add = P.TensorAdd() else: self.matrix_G = Parameter(Tensor( np.eye(out_channels).astype(np.float32)), name="matrix_G", requires_grad=False) self.abs = P.Abs() self.reduce_max = P.ReduceMax(keep_dims=False) self.neg = P.Neg() self.reduce_sum = P.ReduceSum() self.matmul = P.MatMul(transpose_b=True) self.cube_matmul = P.CusMatMulCube(transpose_a=True) self.cast = P.Cast() self.is_nsp_layer = (out_channels == 2) else: self.is_Ascend = False self.matrix_G = Parameter(Tensor( np.eye(out_channels).astype(np.float32)), name="matrix_G", requires_grad=False) self.cube_matmul = P.MatMul(transpose_a=True) self.getG = P.InsertGradientOf(self.save_gradient)
def __init__(self): super().__init__() self.matmul = P.MatMul() self.add = P.TensorAdd()
def __init__(self, in_channel, out_channel, rate, stride=1, num=0, index=None): super(ResidualBlock, self).__init__() channel = out_channel // self.expansion prune_channel = math.ceil(channel * rate) self.conv1 = _conv1x1(in_channel, prune_channel, stride=1) self.bn1 = _bn(prune_channel) self.conv2 = _conv3x3(prune_channel, prune_channel, stride=stride) self.bn2 = _bn(prune_channel) self.conv3 = _conv1x1(prune_channel, math.ceil(channel * rate * self.expansion), stride=1) self.bn3 = _bn_last(math.ceil(channel * rate * self.expansion)) self.relu = nn.ReLU() self.down_sample = False if stride != 1 or in_channel != out_channel: self.down_sample = True self.down_sample_layer = None if self.down_sample: self.down_sample_layer = nn.SequentialCell( [_conv1x1(in_channel, out_channel, stride), _bn(out_channel)]) self.add = P.TensorAdd() self.op = P.ScatterNd() self.transpose = P.Transpose() self.idx = None self.shape = None if out_channel == 256: self.shape = (out_channel, 1, 56, 56) if num == 0: self.idx = index[0] elif num == 1: self.idx = index[1] elif num == 2: self.idx = index[2] elif out_channel == 512: self.shape = (out_channel, 1, 28, 28) if num == 0: self.idx = index[3] elif num == 1: self.idx = index[4] elif num == 2: self.idx = index[5] elif num == 3: self.idx = index[6] elif out_channel == 1024: self.shape = (out_channel, 1, 14, 14) if num == 0: self.idx = index[7] elif num == 1: self.idx = index[8] elif num == 2: self.idx = index[9] elif num == 3: self.idx = index[10] elif num == 4: self.idx = index[11] elif num == 5: self.idx = index[12] elif out_channel == 2048: self.shape = (out_channel, 1, 7, 7) if num == 0: self.idx = index[13] elif num == 1: self.idx = index[14] elif num == 2: self.idx = index[15]
def __init__(self): super(SecondNet, self).__init__() self.add = P.TensorAdd()
def __init__(self, in_channels, out_channels, stride=1, down_sample=False, momentum=0.1, training=False, weights_update=False): super(ResidualBlockUsing, self).__init__() self.affine = weights_update out_chls = out_channels // self.expansion self.conv1 = _conv(in_channels, out_chls, kernel_size=1, stride=1, padding=0) self.bn1 = _BatchNorm2dInit(out_chls, momentum=momentum, affine=self.affine, use_batch_statistics=training) self.conv2 = _conv(out_chls, out_chls, kernel_size=3, stride=stride, padding=1) self.bn2 = _BatchNorm2dInit(out_chls, momentum=momentum, affine=self.affine, use_batch_statistics=training) self.conv3 = _conv(out_chls, out_channels, kernel_size=1, stride=1, padding=0) self.bn3 = _BatchNorm2dInit(out_channels, momentum=momentum, affine=self.affine, use_batch_statistics=training) if training: self.bn1 = self.bn1.set_train() self.bn2 = self.bn2.set_train() self.bn3 = self.bn3.set_train() if not weights_update: self.conv1.weight.requires_grad = False self.conv2.weight.requires_grad = False self.conv3.weight.requires_grad = False self.relu = P.ReLU() self.downsample = down_sample if self.downsample: self.conv_down_sample = _conv(in_channels, out_channels, kernel_size=1, stride=stride, padding=0) self.bn_down_sample = _BatchNorm2dInit( out_channels, momentum=momentum, affine=self.affine, use_batch_statistics=training) if training: self.bn_down_sample = self.bn_down_sample.set_train() if not weights_update: self.conv_down_sample.weight.requires_grad = False self.add = P.TensorAdd()
def __init__(self, dropout_prob=0.1): super(ResidualConnection, self).__init__() self.add = P.TensorAdd() self.dropout = nn.Dropout(1.0 - dropout_prob)
def __init__(self, args, strategy): super(SemiAutoOneHotNet, self).__init__() self.a = args.a self.b = args.b self.c = args.c self.d = args.d self.e = args.e self.cast = P.Cast() self.cast.set_strategy(strategy=strategy.twod_strategy) self.cast1 = P.Cast() self.cast1.set_strategy(strategy=strategy.twod_strategy) self.cast2 = P.Cast() self.cast2.set_strategy(strategy=strategy.twod_strategy) self.cast3 = P.Cast() self.cast3.set_strategy(strategy=strategy.scalar_strategy) self.cast4 = P.Cast() self.cast4.set_strategy(strategy=strategy.scalar_strategy) self.a_const = Tensor(self.a, dtype=mstype.float32) self.b_const = Tensor(self.b, dtype=mstype.float32) self.c_const = Tensor(self.c, dtype=mstype.float32) self.d_const = Tensor(self.d, dtype=mstype.float32) self.e_const = Tensor(self.e, dtype=mstype.float32) self.m_const_zero = Tensor(0, dtype=mstype.float32) self.a_const_one = Tensor(1, dtype=mstype.float32) self.onehot = P.OneHot() self.onehot.set_strategy(strategy=strategy.onehot_strategy) self.exp = P.Exp() self.exp.set_strategy(strategy=strategy.twod_strategy) self.exp2 = P.Exp() self.exp2.set_strategy(strategy=strategy.twod_strategy) self.exp3 = P.Exp() self.exp3.set_strategy(strategy=strategy.twod_strategy) self.mul_const = P.Mul() self.mul_const.set_strategy(strategy=strategy.scalar_twod_strategy) self.mul_const2 = P.TensorAdd() self.mul_const2.set_strategy(strategy=strategy.scalar_twod_strategy) self.mul_const3 = P.Sub() self.mul_const3.set_strategy(strategy=strategy.twod_scalar_strategy) self.mul_const4 = P.Sub() self.mul_const4.set_strategy(strategy=strategy.scalar_twod_strategy) self.mul_const5 = P.Mul() self.mul_const5.set_strategy(strategy=strategy.twod_scalar_strategy) self.mul = P.Mul() self.mul.set_strategy(strategy=strategy.twod_twod_strategy) self.mul2 = P.Mul() self.mul2.set_strategy(strategy=strategy.twod_twod_strategy) self.mul3 = P.TensorAdd() self.mul3.set_strategy(strategy=strategy.twod_twod_strategy) self.mul4 = P.Sub() self.mul4.set_strategy(strategy=strategy.twod_twodbc_strategy) self.mul5 = P.RealDiv() self.mul5.set_strategy(strategy=strategy.twod_twodbc_strategy) self.mul6 = P.Mul() self.mul6.set_strategy(strategy=strategy.twod_twod_strategy) self.mul7 = P.Mul() self.mul7.set_strategy(strategy=strategy.twod_scalar_strategy) self.mul8 = P.RealDiv() self.mul8.set_strategy(strategy=strategy.scalar_scalar_strategy) self.mul9 = P.TensorAdd() self.mul9.set_strategy(strategy=strategy.twod_scalar_strategy) self.reduce_max = P.ReduceMax(keep_dims=True) self.reduce_max.set_strategy(strategy=strategy.twod_strategy) self.reduce_sum = P.ReduceSum(keep_dims=False) self.reduce_sum.set_strategy(strategy=strategy.twod_strategy) self.reduce_sum_2 = P.ReduceSum(keep_dims=False) self.reduce_sum_2.set_strategy(strategy=strategy.twod_strategy) self.reduce_sum_3 = P.ReduceSum(keep_dims=False) self.reduce_sum_3.set_strategy(strategy=strategy.oned_strategy) self.reshape = P.Reshape() self.log = P.Log() self.log.set_strategy(strategy=strategy.twod_strategy) self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.normalize = P.L2Normalize(axis=1) self.normalize.set_strategy(strategy=strategy.twod_strategy_m) self.normalize2 = P.L2Normalize(axis=1) self.normalize2.set_strategy(strategy=strategy.twod_strategy_m) self.fc = P.MatMul(transpose_b=True) self.fc.set_strategy(strategy=strategy.twodbc_twod_strategy) weight_shape = [args.num_classes, args.emb_size] weight_np = np.zeros(weight_shape, np.float32) self.weight = Parameter(Tensor(weight_np), name='model_parallel_weight')
def __init__(self): super(SecondNet, self).__init__() self.addN = P.AddN() self.max = P.Maximum() self.add = P.TensorAdd()
def __init__(self, in_features, out_features): super(Net, self).__init__() self.weight = Parameter(Tensor(np.ones([out_features, in_features]).astype(np.float32)), name="weight") self.bias = Parameter(Tensor(np.ones([out_features]).astype(np.float32)), name="bias") self.matmul = P.MatMul() self.add = P.TensorAdd()
def __init__(self, target_shape, strategy0=None, strategy1=None): super(Reshape, self).__init__() self.add = P.TensorAdd(strategy=strategy0) self.reshape = P.Reshape(strategy=strategy1) self.shape = tuple(target_shape)
# ============================================================================ """ test_hypermap """ import numpy as np from mindspore import Tensor from mindspore.common.api import ms_function from mindspore.ops import Primitive from mindspore.ops import composite as C from mindspore.ops import functional as F from mindspore.ops import operations as P from ...ut_filter import non_graph_engine # pylint: disable=W0613 # W0613: unused-argument tensor_add = P.TensorAdd() scala_add = Primitive('scalar_add') add = C.MultitypeFuncGraph('add') @add.register("Number", "Number") def add_scala(x, y): return scala_add(x, y) @add.register("Tensor", "Tensor") def add_tensor(x, y): return tensor_add(x, y) hyper_add = C.HyperMap(add)
# you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ from mindspore.ops import Primitive from mindspore.ops import operations as P tuple_getitem = Primitive('tuple_getitem') add = P.TensorAdd() max_pool = P.MaxPoolWithArgmax(pad_mode="same", kernel_size=3, strides=2) make_tuple = Primitive('make_tuple') transdata = Primitive("TransData") Transpose = P.Transpose() class FnDict: def __init__(self): self.fnDict = {} def __call__(self, fn): self.fnDict[fn.__name__] = fn def __getitem__(self, name): return self.fnDict[name]
def __init__(self, axis=0, epsilon=1e-4, strategy0=None, strategy1=None): super(L2normalize, self).__init__() self.add = P.TensorAdd(strategy=strategy0) self.l2norm = P.L2Normalize(axis, epsilon, strategy1)
def __init__(self, vocab_size, embedding_size, field_size, param_init='normal', target='CPU', slice_mode='batch_slice', feature_num_list=None, max_norm=None, sparse=True, operator='SUM'): super(MultiFieldEmbeddingLookup, self).__init__(vocab_size, embedding_size, param_init, target, slice_mode, feature_num_list, max_norm, sparse) self.field_size = validator.check_positive_int(field_size, 'field_size') self.operator = operator self.mul = P.Mul() self.inf_mask_mul = P.Mul() self.bias_add = P.TensorAdd() self.inf_add = P.TensorAdd() self.merge_op = None self.count_op = P.UnsortedSegmentSum() self.abs = P.Abs() self.equal = P.Equal() self.add = P.TensorAdd() self.cast = P.Cast() self.div_no_nan = P.DivNoNan() self.expand = P.ExpandDims() self.max_mask_mul = P.Mul() self.max_no_equal = P.NotEqual() if operator == MultiFieldEmbeddingLookup.OPERATOR_SUM: self.merge_op = P.UnsortedSegmentSum() elif operator == MultiFieldEmbeddingLookup.OPERATOR_MAX: self.merge_op = P.UnsortedSegmentMax() elif operator == MultiFieldEmbeddingLookup.OPERATOR_MEAN: self.merge_op = P.UnsortedSegmentSum() else: raise ValueError( "The operator supports ['SUM', 'MAX', 'MEAN'], but found: " + str(operator)) parallel_mode = _get_parallel_mode() is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) if slice_mode in ["table_row_slice", "batch_slice" ] and is_auto_parallel: self.merge_op.shard( ((get_group_size(), 1, 1), (get_group_size(), 1))) self.expand.shard(((get_group_size(), ), )) self.bias_add.shard(((1, 1), (1, 1))) self.mul.shard( ((get_group_size(), 1, 1), (get_group_size(), 1, 1))) self.count_op.shard(((get_group_size(), 1), (get_group_size(), 1))) self.add.shard(((get_group_size(), ), (get_group_size(), ))) self.div_no_nan.shard( ((get_group_size(), 1), (get_group_size(), 1))) self.max_mask_mul.shard( ((get_group_size(), 1), (get_group_size(), 1))) self.max_no_equal.shard(((1, ), ())) if operator == MultiFieldEmbeddingLookup.OPERATOR_MAX: self.equal.shard(((get_group_size(), 1, 1), ())) self.inf_mask_mul.shard(((get_group_size(), 1, 1), ())) self.merge_op.shard( ((get_group_size(), 1), (get_group_size(), ))) self.count_op.shard( ((get_group_size(), ), (get_group_size(), ))) self.inf_add.shard( ((get_group_size(), 1, 1), (get_group_size(), 1, 1))) elif slice_mode == "table_column_slice" and is_auto_parallel: self.merge_op.shard(((1, 1, get_group_size()), (1, 1))) self.div_no_nan.shard(((1, get_group_size()), (1, 1))) self.bias_add.shard(((1, 1), (1, 1))) self.mul.shard(((1, 1, 1), (1, 1, get_group_size()))) self.count_op.shard(((1, 1), (1, 1))) self.add.shard(((1, ), (1, ))) self.max_mask_mul.shard(((1, get_group_size()), (1, 1))) self.expand.shard(((1, ), )) self.max_no_equal.shard(((1, ), ())) if operator == MultiFieldEmbeddingLookup.OPERATOR_MAX: self.equal.shard(((1, 1, 1), ())) self.inf_mask_mul.shard(((1, 1, 1), ())) self.merge_op.shard(((1, get_group_size()), (1, ))) self.count_op.shard(((1, ), (1, ))) self.inf_add.shard(((1, 1, get_group_size()), (1, 1, 1))) else: if is_auto_parallel: raise ValueError( "slice_mode should be ['table_row_slice', 'batch_slice' and \ 'table_column_slice'], but get " + str(slice_mode)) # Min value for fp32 self.negative_inf_value = -3.402823466E+38
def __init__(self): super(MultiOutNet, self).__init__() self.add = P.TensorAdd() self.mul = P.Mul() self.sum = P.ReduceSum()
def __init__(self, in_channels, out_channels, weight_init='normal', bias_init='zeros', damping=0.03, loss_scale=1, frequency=278, batch_size=32.0, has_bias=True, activation=None): super(Dense_ThorNoBN, self).__init__() self.batch_size = batch_size self.in_channels = check_int_positive(in_channels) self.out_channels = check_int_positive(out_channels) self.has_bias = check_bool(has_bias) self.thor = True if isinstance(weight_init, Tensor): if weight_init.dim() != 2 or weight_init.shape()[0] != out_channels or \ weight_init.shape()[1] != in_channels: raise ValueError("weight_init shape error") self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") if self.has_bias: if isinstance(bias_init, Tensor): if bias_init.dim() != 1 or bias_init.shape( )[0] != out_channels: raise ValueError("bias_init shape error") self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") self.matmul = P.MatMul(transpose_b=True) self.bias_add = P.BiasAdd() self.activation = get_activation(activation) self.activation_flag = self.activation is not None self.matrix_A_inv = Parameter(Tensor( np.zeros([128, 128, 16, 16]).astype(np.float16)), name='matrix_A_inv', requires_grad=False) self.matrix_G_inv = Parameter(Tensor( np.zeros([63, 63, 16, 16]).astype(np.float16)), name="matrix_G_inv", requires_grad=False) self.fake_G = Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)) self.matmul = P.MatMul(transpose_b=True) self.cube_matmul = P.CusMatMulCube(transpose_a=True) self.matrix_combine = P.CusMatrixCombine() self.cholesky = P.CusCholeskyTrsm() self.shape = P.Shape() self.reshape = P.Reshape() self.transpose = P.Transpose() self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) self.mul = P.Mul() self.cast = P.Cast() self.damping = Tensor(damping) self.loss_scale = Tensor(1 / loss_scale, mstype.float16) self.vector_matmul = P.CusBatchMatMul() self.pad = P.Pad(((0, 24), (0, 24))) self.pad1 = P.Pad(((0, 8), (0, 8))) self.slice = P.Slice() self.gather = P.GatherV2() self.assignadd = P.AssignAdd() self.freq = Tensor(frequency, mstype.int32) self.axis = 0 self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) self.fused_abs_max1 = P.CusFusedAbsMax1([1000, 1000]) self.fused_abs_max2 = P.CusFusedAbsMax1() self.log = P.Log() self.exp = P.Exp() self.dampingA = Tensor(np.identity(2048), mstype.float32) self.dampingG = Tensor(np.identity(1024), mstype.float32) self.add = P.TensorAdd() self.sqrt = P.Sqrt() self.getG = P.InsertGradientOf(self.save_gradient) self.fake_G_inv_max = Tensor(np.zeros([ 1, ]).astype(np.float32)) self.norm = P.L2Normalize(axis=1) self.multiplier = Parameter(Tensor(np.ones([1]), dtype=mstype.float16), requires_grad=True, name='multiplier')
def __init__(self): super(FirstNet, self).__init__() self.net = SecondNet().add_flags_recursive(fp16=True) self.add = P.TensorAdd()
def __init__(self, strategy1, strategy2): super().__init__() self.matmul = P.MatMul().shard(strategy1) self.add = P.TensorAdd().shard(strategy2)
def __init__(self, ): super(SummaryDemo, self).__init__() self.s = P.ScalarSummary() self.histogram_summary = P.HistogramSummary() self.add = P.TensorAdd()