def __init__(self, num_classes, num_boxes, neg_pre_positive, batch_size): super(MultiBoxLoss, self).__init__() self.num_classes = num_classes self.num_boxes = num_boxes self.neg_pre_positive = neg_pre_positive self.notequal = P.NotEqual() self.less = P.Less() self.tile = P.Tile() self.reduce_sum = P.ReduceSum() self.reduce_mean = P.ReduceMean() self.expand_dims = P.ExpandDims() self.smooth_l1_loss = P.SmoothL1Loss() self.cross_entropy = SoftmaxCrossEntropyWithLogits() self.maximum = P.Maximum() self.minimum = P.Minimum() self.sort_descend = P.TopK(True) self.sort = P.TopK(True) self.gather = P.GatherNd() self.max = P.ReduceMax() self.log = P.Log() self.exp = P.Exp() self.concat = P.Concat(axis=1) self.reduce_sum2 = P.ReduceSum(keep_dims=True) self.idx = Tensor( np.reshape(np.arange(batch_size * num_boxes), (-1, 1)), ms.int32)
def __init__(self, network, optimizer, scale_update_cell=None, accumulation_steps=1, enable_global_norm=False): super(BertTrainAccumulateStepsWithLossScaleCell, self).__init__(auto_prefix=False) self.network = network self.network.set_grad() self.weights = optimizer.parameters self.optimizer = optimizer self.accumulation_steps = accumulation_steps self.enable_global_norm = enable_global_norm self.one = Tensor(np.array([1]).astype(np.int32)) self.zero = Tensor(np.array([0]).astype(np.int32)) self.local_step = Parameter(initializer(0, [1], mstype.int32), name="local_step") self.accu_grads = self.weights.clone(prefix="accu_grads", init='zeros') self.accu_overflow = Parameter(initializer(0, [1], mstype.int32), name="accu_overflow") self.loss = Parameter(initializer(0, [1], mstype.float32), name="accu_loss") self.grad = C.GradOperation(get_by_list=True, sens_param=True) self.reducer_flag = False self.parallel_mode = context.get_auto_parallel_context("parallel_mode") if self.parallel_mode in [ ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL ]: self.reducer_flag = True self.grad_reducer = F.identity self.degree = 1 if self.reducer_flag: self.degree = get_group_size() self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree) self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) self.overflow_reducer = F.identity if self.is_distributed: self.overflow_reducer = P.AllReduce() self.cast = P.Cast() self.alloc_status = P.NPUAllocFloatStatus() self.get_status = P.NPUGetFloatStatus() self.clear_before_grad = P.NPUClearFloatStatus() self.reduce_sum = P.ReduceSum(keep_dims=False) self.base = Tensor(1, mstype.float32) self.less_equal = P.LessEqual() self.logical_or = P.LogicalOr() self.not_equal = P.NotEqual() self.select = P.Select() self.reshape = P.Reshape() self.hyper_map = C.HyperMap() self.loss_scale = None self.loss_scaling_manager = scale_update_cell if scale_update_cell: self.loss_scale = Parameter(Tensor( scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale")
def __init__(self, config, network, loss, eos_token=6): super(PANGUALPHAWithLoss, self).__init__(auto_prefix=False) self.network = network self.loss = loss self.eos_token = eos_token self.slice = P.StridedSlice().shard(((config.dp, 1),)) self.not_equal = P.NotEqual().shard(((config.dp, 1), ())) self.batch_size = config.batch_size self.len = config.seq_length self.slice_mask = P.StridedSlice().shard(((config.dp, 1, 1),))
def test_notequal_float(): op = P.NotEqual() op_wrapper = OpNetWrapper(op) input_x = Tensor(np.array([1, 2, 3]).astype(np.float32)) input_y = Tensor(np.array([-1, 0, 3]).astype(np.float32)) outputs = op_wrapper(input_x, input_y) print(outputs) assert np.allclose(outputs.asnumpy(), (True, True, False))
def __init__(self, network, k, num_eval_neg): super(PredictWithSigmoid, self).__init__() self.network = network self.topk = P.TopK(sorted=True) self.squeeze = P.Squeeze() self.k = k self.num_eval_neg = num_eval_neg self.gather = P.Gather() self.reshape = P.Reshape() self.reducesum = P.ReduceSum(keep_dims=False) self.notequal = P.NotEqual()
def __init__(self, num_cls=21, ignore_label=255): super(SoftmaxCrossEntropyLoss, self).__init__() self.one_hot = P.OneHot(axis=-1) self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.cast = P.Cast() self.ce = nn.SoftmaxCrossEntropyWithLogits() self.not_equal = P.NotEqual() self.num_cls = num_cls self.ignore_label = ignore_label self.mul = P.Mul() self.sum = P.ReduceSum(False) self.div = P.RealDiv() self.transpose = P.Transpose() self.reshape = P.Reshape()
def __init__(self, config, network, loss, eos_token=6): super(PANGUALPHAWithLossPipeline, self).__init__(auto_prefix=False) self.network = network self.loss = loss self.eos_token = eos_token self.slice = P.StridedSlice().shard(((config.dp, 1),)) self.not_equal = P.NotEqual().shard(((config.dp, 1), ())) self.batch_size = config.batch_size self.len = config.seq_length self.micro_batch_step = config.micro_size self.micro_input = nn.CellList() self.slice_mask = P.StridedSlice().shard(((config.dp, 1, 1),)) for i in range(self.micro_batch_step): micro = MicroBatch(config) micro.micro_slice.add_prim_attr("micro", i) micro.micro_slice.add_prim_attr("start", i) self.micro_input.append(micro)
def __init__(self, num, ignore_label): super(OhemLoss, self).__init__() self.mul = P.Mul() self.shape = P.Shape() self.one_hot = nn.OneHot(-1, num, 1.0, 0.0) self.squeeze = P.Squeeze() self.num = num self.cross_entropy = P.SoftmaxCrossEntropyWithLogits() self.mean = P.ReduceMean() self.select = P.Select() self.reshape = P.Reshape() self.cast = P.Cast() self.not_equal = P.NotEqual() self.equal = P.Equal() self.reduce_sum = P.ReduceSum(keep_dims=False) self.fill = P.Fill() self.transpose = P.Transpose() self.ignore_label = ignore_label self.loss_weight = 1.0
def __init__(self): super(NetNotEqual, self).__init__() self.NotEqual = P.NotEqual()
def _IgammacContinuedFraction(ax, x, a, enabled): """Helper function for computing Igammac using a continued fraction.""" abs_x = P.Abs() logicaland = P.LogicalAnd() greater = P.Greater() less = P.Less() notequal = P.NotEqual() fill = P.Fill() shape = P.Shape() dtype = P.DType() select = P.Select() if dtype(ax) == mstype.float16: epsilon = eps_fp16 else: epsilon = eps_fp32 def cond(vals): enabled = vals[0] c = vals[5] return logicaland(less(c, 2000), enabled) def body(vals): enabled = vals[0] ans = vals[1] t = vals[2] y = vals[3] z = vals[4] c = vals[5] pkm1 = vals[6] qkm1 = vals[7] pkm2 = vals[8] qkm2 = vals[9] dpkm2_da = vals[10] dqkm2_da = vals[11] dpkm1_da = vals[12] dqkm1_da = vals[13] dans_da = vals[14] c = c + 1 y = y + 1 z = z + 2 yc = y * c pk = pkm1 * z - pkm2 * yc qk = qkm1 * z - qkm2 * yc qk_is_nonzero = notequal(qk, 0) r = pk / qk t = select(qk_is_nonzero, abs_x((ans - r) / r), fill(dtype(t), shape(t), 1)) ans = select(qk_is_nonzero, r, ans) dpk_da = dpkm1_da * z - pkm1 - dpkm2_da * yc + pkm2 * c dqk_da = dqkm1_da * z - qkm1 - dqkm2_da * yc + qkm2 * c dans_da_new = select(qk_is_nonzero, (dpk_da - ans * dqk_da) / qk, dans_da) grad_conditional = select(qk_is_nonzero, abs_x(dans_da_new - dans_da), fill(dtype(dans_da), shape(dans_da), 1)) pkm2 = pkm1 pkm1 = pk qkm2 = qkm1 qkm1 = qk dpkm2_da = dpkm1_da dqkm2_da = dqkm1_da dpkm1_da = dpk_da dqkm1_da = dqk_da rescale = greater(abs_x(pk), 1 / epsilon) pkm2 = select(rescale, pkm2 * epsilon, pkm2) pkm1 = select(rescale, pkm1 * epsilon, pkm1) qkm2 = select(rescale, qkm2 * epsilon, qkm2) qkm1 = select(rescale, qkm1 * epsilon, qkm1) dpkm2_da = select(rescale, dpkm2_da * epsilon, dpkm2_da) dqkm2_da = select(rescale, dqkm2_da * epsilon, dqkm2_da) dpkm1_da = select(rescale, dpkm1_da * epsilon, dpkm1_da) dqkm1_da = select(rescale, dqkm1_da * epsilon, dqkm1_da) conditional = logicaland(enabled, greater(grad_conditional, epsilon)) return (conditional, select(enabled, ans, vals[1]), select(enabled, t, vals[2]), select(enabled, y, vals[3]), select(enabled, z, vals[4]), c, select(enabled, pkm1, vals[6]), select(enabled, qkm1, vals[7]), select(enabled, pkm2, vals[8]), select(enabled, qkm2, vals[9]), select(enabled, dpkm2_da, vals[10]), select(enabled, dqkm2_da, vals[11]), select(enabled, dpkm1_da, vals[12]), select(enabled, dqkm1_da, vals[13]), select(enabled, dans_da_new, vals[14])) y = 1 - a z = x + y + 1 c = fill(dtype(x), shape(x), 0) pkm2 = fill(dtype(x), shape(x), 1) qkm2 = x pkm1 = x + 1 qkm1 = z * x ans = pkm1 / qkm1 t = fill(dtype(x), shape(x), 1) dpkm2_da = fill(dtype(x), shape(x), 0) dqkm2_da = fill(dtype(x), shape(x), 0) dpkm1_da = fill(dtype(x), shape(x), 0) dqkm1_da = -x dans_da = (dpkm1_da - ans * dqkm1_da) / qkm1 vals = (enabled, ans, t, y, z, c, pkm1, qkm1, pkm2, qkm2, dpkm2_da, dqkm2_da, dpkm1_da, dqkm1_da, dans_da) vals = _while_helper_func(cond, body, vals) ans = vals[1] return ans * ax
def __init__(self, vocab_size, embedding_size, field_size, param_init='normal', target='CPU', slice_mode='batch_slice', feature_num_list=None, max_norm=None, sparse=True, operator='SUM'): super(MultiFieldEmbeddingLookup, self).__init__(vocab_size, embedding_size, param_init, target, slice_mode, feature_num_list, max_norm, sparse) self.field_size = validator.check_positive_int(field_size, 'field_size') self.operator = operator self.mul = P.Mul() self.inf_mask_mul = P.Mul() self.bias_add = P.Add() self.inf_add = P.Add() self.merge_op = None self.count_op = P.UnsortedSegmentSum() self.abs = P.Abs() self.equal = P.Equal() self.add = P.Add() self.cast = P.Cast() self.div_no_nan = P.DivNoNan() self.expand = P.ExpandDims() self.max_mask_mul = P.Mul() self.max_no_equal = P.NotEqual() if operator == MultiFieldEmbeddingLookup.OPERATOR_SUM: self.merge_op = P.UnsortedSegmentSum() elif operator == MultiFieldEmbeddingLookup.OPERATOR_MAX: self.merge_op = P.UnsortedSegmentMax() elif operator == MultiFieldEmbeddingLookup.OPERATOR_MEAN: self.merge_op = P.UnsortedSegmentSum() else: raise ValueError( "The operator supports ['SUM', 'MAX', 'MEAN'], but found: " + str(operator)) parallel_mode = _get_parallel_mode() is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) if slice_mode in ["table_row_slice", "batch_slice" ] and is_auto_parallel: self.merge_op.shard( ((get_group_size(), 1, 1), (get_group_size(), 1))) self.expand.shard(((get_group_size(), ), )) self.bias_add.shard(((1, 1), (1, 1))) self.mul.shard( ((get_group_size(), 1, 1), (get_group_size(), 1, 1))) self.count_op.shard(((get_group_size(), 1), (get_group_size(), 1))) self.add.shard(((get_group_size(), ), (get_group_size(), ))) self.div_no_nan.shard( ((get_group_size(), 1), (get_group_size(), 1))) self.max_mask_mul.shard( ((get_group_size(), 1), (get_group_size(), 1))) self.max_no_equal.shard(((1, ), ())) if operator == MultiFieldEmbeddingLookup.OPERATOR_MAX: self.equal.shard(((get_group_size(), 1, 1), ())) self.inf_mask_mul.shard(((get_group_size(), 1, 1), ())) self.merge_op.shard( ((get_group_size(), 1), (get_group_size(), ))) self.count_op.shard( ((get_group_size(), ), (get_group_size(), ))) self.inf_add.shard( ((get_group_size(), 1, 1), (get_group_size(), 1, 1))) elif slice_mode == "table_column_slice" and is_auto_parallel: self.merge_op.shard(((1, 1, get_group_size()), (1, 1))) self.div_no_nan.shard(((1, get_group_size()), (1, 1))) self.bias_add.shard(((1, 1), (1, 1))) self.mul.shard(((1, 1, 1), (1, 1, get_group_size()))) self.count_op.shard(((1, 1), (1, 1))) self.add.shard(((1, ), (1, ))) self.max_mask_mul.shard(((1, get_group_size()), (1, 1))) self.expand.shard(((1, ), )) self.max_no_equal.shard(((1, ), ())) if operator == MultiFieldEmbeddingLookup.OPERATOR_MAX: self.equal.shard(((1, 1, 1), ())) self.inf_mask_mul.shard(((1, 1, 1), ())) self.merge_op.shard(((1, get_group_size()), (1, ))) self.count_op.shard(((1, ), (1, ))) self.inf_add.shard(((1, 1, get_group_size()), (1, 1, 1))) else: if is_auto_parallel: raise ValueError( "slice_mode should be ['table_row_slice', 'batch_slice' and \ 'table_column_slice'], but get " + str(slice_mode)) # Min value for fp32 self.negative_inf_value = -3.402823466E+38
'desc_inputs': [[4]], 'desc_bprop': [[4]]}), ('RealDiv', { 'block': P.RealDiv(), 'desc_inputs': [[4, 5], [2, 3, 4, 5]], 'desc_bprop': [[2, 3, 4, 5]]}), ('Div', { 'block': P.Div(), 'desc_inputs': [[4, 5], [2, 3, 4, 5]], 'desc_bprop': [[2, 3, 4, 5]]}), ('Equal', { 'block': P.Equal(), 'desc_inputs': [[3, 4, 5], [4, 5]], 'desc_bprop': [Tensor(np.zeros((3, 4, 5), np.bool_))]}), ('NotEqual', { 'block': P.NotEqual(), 'desc_inputs': [[4, 1], [2, 3, 4, 5]], 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}), ('NotEqual_0', { 'block': P.NotEqual(), 'desc_inputs': [ 1, [2, 3, 4, 5]], 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))], 'skip': ['backward']}), ('Greater', { 'block': P.Greater(), 'desc_inputs': [[2, 3, 4, 1], [4, 5]], 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}), ('GreaterEqual', { 'block': P.GreaterEqual(), 'desc_inputs': [[2, 3, 4, 1], [4, 5]], 'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}),
# input is not tensor ('EqualCount0', { 'block': (P.EqualCount(), {'exception': TypeError, 'error_keywords': ['EqualCount']}), 'desc_inputs': [5.0, Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # type of x and y not match ('EqualCount1', { 'block': (P.EqualCount(), {'exception': TypeError, 'error_keywords': ['EqualCount']}), 'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # shape of x and y not match # input is not tensor ('NotEqual0', { 'block': (P.NotEqual(), {'exception': TypeError, 'error_keywords': ['NotEqual']}), 'desc_inputs': [5.0, Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # type of x and y not match ('NotEqual1', { 'block': (P.NotEqual(), {'exception': TypeError, 'error_keywords': ['NotEqual']}), 'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32))], 'skip': ['backward']}), # shape of x and y not match ('NotEqual2', { 'block': (P.NotEqual(), {'exception': ValueError, 'error_keywords': ['NotEqual']}), 'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32))], 'skip': ['backward']}), # input is not tensor ('Greater0', {
('EqualCount1', { 'block': (P.EqualCount(), { 'exception': TypeError, 'error_keywords': ['EqualCount'] }), 'desc_inputs': [ Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32)) ], 'skip': ['backward'] }), # shape of x and y not match # shape of x and y not match ('NotEqual2', { 'block': (P.NotEqual(), { 'exception': ValueError, 'error_keywords': ['NotEqual'] }), 'desc_inputs': [ Tensor(np.ones([3, 4]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32)) ], 'skip': ['backward'] }), # shape of x and y not match ('Greater2', { 'block': (P.Greater(), { 'exception': ValueError, 'error_keywords': ['Greater']
def __init__(self, strategy1, strategy2): super().__init__() self.matmul = P.MatMul().set_strategy(strategy1) self.notequal = P.NotEqual().set_strategy(strategy2)
def __init__(self, strategy1, strategy2): super().__init__() self.matmul = P.MatMul().shard(strategy1) self.equal = P.Equal().shard(strategy2) self.notequal = P.NotEqual().shard(strategy2) self.logical = P.LogicalOr().shard(strategy2)