def __init__(self, num_classes, num_boxes, neg_pre_positive, batch_size):
super(MultiBoxLoss, self).__init__()
self.num_classes = num_classes
self.num_boxes = num_boxes
self.neg_pre_positive = neg_pre_positive
self.notequal = P.NotEqual()
self.less = P.Less()
self.tile = P.Tile()
self.reduce_sum = P.ReduceSum()
self.reduce_mean = P.ReduceMean()
self.expand_dims = P.ExpandDims()
self.smooth_l1_loss = P.SmoothL1Loss()
self.cross_entropy = SoftmaxCrossEntropyWithLogits()
self.maximum = P.Maximum()
self.minimum = P.Minimum()
self.sort_descend = P.TopK(True)
self.sort = P.TopK(True)
self.gather = P.GatherNd()
self.max = P.ReduceMax()
self.log = P.Log()
self.exp = P.Exp()
self.concat = P.Concat(axis=1)
self.reduce_sum2 = P.ReduceSum(keep_dims=True)
self.idx = Tensor(
np.reshape(np.arange(batch_size * num_boxes), (-1, 1)), ms.int32)
def __init__(self,
network,
optimizer,
scale_update_cell=None,
accumulation_steps=1,
enable_global_norm=False):
super(BertTrainAccumulateStepsWithLossScaleCell,
self).__init__(auto_prefix=False)
self.network = network
self.network.set_grad()
self.weights = optimizer.parameters
self.optimizer = optimizer
self.accumulation_steps = accumulation_steps
self.enable_global_norm = enable_global_norm
self.one = Tensor(np.array([1]).astype(np.int32))
self.zero = Tensor(np.array([0]).astype(np.int32))
self.local_step = Parameter(initializer(0, [1], mstype.int32),
name="local_step")
self.accu_grads = self.weights.clone(prefix="accu_grads", init='zeros')
self.accu_overflow = Parameter(initializer(0, [1], mstype.int32),
name="accu_overflow")
self.loss = Parameter(initializer(0, [1], mstype.float32),
name="accu_loss")
self.grad = C.GradOperation(get_by_list=True, sens_param=True)
self.reducer_flag = False
self.parallel_mode = context.get_auto_parallel_context("parallel_mode")
if self.parallel_mode in [
ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL
]:
self.reducer_flag = True
self.grad_reducer = F.identity
self.degree = 1
if self.reducer_flag:
self.degree = get_group_size()
self.grad_reducer = DistributedGradReducer(optimizer.parameters,
False, self.degree)
self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
self.overflow_reducer = F.identity
if self.is_distributed:
self.overflow_reducer = P.AllReduce()
self.cast = P.Cast()
self.alloc_status = P.NPUAllocFloatStatus()
self.get_status = P.NPUGetFloatStatus()
self.clear_before_grad = P.NPUClearFloatStatus()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.base = Tensor(1, mstype.float32)
self.less_equal = P.LessEqual()
self.logical_or = P.LogicalOr()
self.not_equal = P.NotEqual()
self.select = P.Select()
self.reshape = P.Reshape()
self.hyper_map = C.HyperMap()
self.loss_scale = None
self.loss_scaling_manager = scale_update_cell
if scale_update_cell:
self.loss_scale = Parameter(Tensor(
scale_update_cell.get_loss_scale(), dtype=mstype.float32),
name="loss_scale")
def __init__(self, config, network, loss, eos_token=6):
super(PANGUALPHAWithLoss, self).__init__(auto_prefix=False)
self.network = network
self.loss = loss
self.eos_token = eos_token
self.slice = P.StridedSlice().shard(((config.dp, 1),))
self.not_equal = P.NotEqual().shard(((config.dp, 1), ()))
self.batch_size = config.batch_size
self.len = config.seq_length
self.slice_mask = P.StridedSlice().shard(((config.dp, 1, 1),))
def test_notequal_float():
op = P.NotEqual()
op_wrapper = OpNetWrapper(op)
input_x = Tensor(np.array([1, 2, 3]).astype(np.float32))
input_y = Tensor(np.array([-1, 0, 3]).astype(np.float32))
outputs = op_wrapper(input_x, input_y)
print(outputs)
assert np.allclose(outputs.asnumpy(), (True, True, False))
def __init__(self, network, k, num_eval_neg):
super(PredictWithSigmoid, self).__init__()
self.network = network
self.topk = P.TopK(sorted=True)
self.squeeze = P.Squeeze()
self.k = k
self.num_eval_neg = num_eval_neg
self.gather = P.Gather()
self.reshape = P.Reshape()
self.reducesum = P.ReduceSum(keep_dims=False)
self.notequal = P.NotEqual()
def __init__(self, num_cls=21, ignore_label=255):
super(SoftmaxCrossEntropyLoss, self).__init__()
self.one_hot = P.OneHot(axis=-1)
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.cast = P.Cast()
self.ce = nn.SoftmaxCrossEntropyWithLogits()
self.not_equal = P.NotEqual()
self.num_cls = num_cls
self.ignore_label = ignore_label
self.mul = P.Mul()
self.sum = P.ReduceSum(False)
self.div = P.RealDiv()
self.transpose = P.Transpose()
self.reshape = P.Reshape()
def __init__(self, config, network, loss, eos_token=6):
super(PANGUALPHAWithLossPipeline, self).__init__(auto_prefix=False)
self.network = network
self.loss = loss
self.eos_token = eos_token
self.slice = P.StridedSlice().shard(((config.dp, 1),))
self.not_equal = P.NotEqual().shard(((config.dp, 1), ()))
self.batch_size = config.batch_size
self.len = config.seq_length
self.micro_batch_step = config.micro_size
self.micro_input = nn.CellList()
self.slice_mask = P.StridedSlice().shard(((config.dp, 1, 1),))
for i in range(self.micro_batch_step):
micro = MicroBatch(config)
micro.micro_slice.add_prim_attr("micro", i)
micro.micro_slice.add_prim_attr("start", i)
self.micro_input.append(micro)
def __init__(self, num, ignore_label):
super(OhemLoss, self).__init__()
self.mul = P.Mul()
self.shape = P.Shape()
self.one_hot = nn.OneHot(-1, num, 1.0, 0.0)
self.squeeze = P.Squeeze()
self.num = num
self.cross_entropy = P.SoftmaxCrossEntropyWithLogits()
self.mean = P.ReduceMean()
self.select = P.Select()
self.reshape = P.Reshape()
self.cast = P.Cast()
self.not_equal = P.NotEqual()
self.equal = P.Equal()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.fill = P.Fill()
self.transpose = P.Transpose()
self.ignore_label = ignore_label
self.loss_weight = 1.0
def __init__(self):
super(NetNotEqual, self).__init__()
self.NotEqual = P.NotEqual()
def _IgammacContinuedFraction(ax, x, a, enabled):
"""Helper function for computing Igammac using a continued fraction."""
abs_x = P.Abs()
logicaland = P.LogicalAnd()
greater = P.Greater()
less = P.Less()
notequal = P.NotEqual()
fill = P.Fill()
shape = P.Shape()
dtype = P.DType()
select = P.Select()
if dtype(ax) == mstype.float16:
epsilon = eps_fp16
else:
epsilon = eps_fp32
def cond(vals):
enabled = vals[0]
c = vals[5]
return logicaland(less(c, 2000), enabled)
def body(vals):
enabled = vals[0]
ans = vals[1]
t = vals[2]
y = vals[3]
z = vals[4]
c = vals[5]
pkm1 = vals[6]
qkm1 = vals[7]
pkm2 = vals[8]
qkm2 = vals[9]
dpkm2_da = vals[10]
dqkm2_da = vals[11]
dpkm1_da = vals[12]
dqkm1_da = vals[13]
dans_da = vals[14]
c = c + 1
y = y + 1
z = z + 2
yc = y * c
pk = pkm1 * z - pkm2 * yc
qk = qkm1 * z - qkm2 * yc
qk_is_nonzero = notequal(qk, 0)
r = pk / qk
t = select(qk_is_nonzero, abs_x((ans - r) / r), fill(dtype(t), shape(t), 1))
ans = select(qk_is_nonzero, r, ans)
dpk_da = dpkm1_da * z - pkm1 - dpkm2_da * yc + pkm2 * c
dqk_da = dqkm1_da * z - qkm1 - dqkm2_da * yc + qkm2 * c
dans_da_new = select(qk_is_nonzero, (dpk_da - ans * dqk_da) / qk, dans_da)
grad_conditional = select(qk_is_nonzero,
abs_x(dans_da_new - dans_da),
fill(dtype(dans_da), shape(dans_da), 1))
pkm2 = pkm1
pkm1 = pk
qkm2 = qkm1
qkm1 = qk
dpkm2_da = dpkm1_da
dqkm2_da = dqkm1_da
dpkm1_da = dpk_da
dqkm1_da = dqk_da
rescale = greater(abs_x(pk), 1 / epsilon)
pkm2 = select(rescale, pkm2 * epsilon, pkm2)
pkm1 = select(rescale, pkm1 * epsilon, pkm1)
qkm2 = select(rescale, qkm2 * epsilon, qkm2)
qkm1 = select(rescale, qkm1 * epsilon, qkm1)
dpkm2_da = select(rescale, dpkm2_da * epsilon, dpkm2_da)
dqkm2_da = select(rescale, dqkm2_da * epsilon, dqkm2_da)
dpkm1_da = select(rescale, dpkm1_da * epsilon, dpkm1_da)
dqkm1_da = select(rescale, dqkm1_da * epsilon, dqkm1_da)
conditional = logicaland(enabled, greater(grad_conditional, epsilon))
return (conditional, select(enabled, ans, vals[1]), select(enabled, t, vals[2]),
select(enabled, y, vals[3]), select(enabled, z, vals[4]),
c, select(enabled, pkm1, vals[6]),
select(enabled, qkm1, vals[7]), select(enabled, pkm2, vals[8]),
select(enabled, qkm2, vals[9]), select(enabled, dpkm2_da, vals[10]),
select(enabled, dqkm2_da, vals[11]), select(enabled, dpkm1_da, vals[12]),
select(enabled, dqkm1_da, vals[13]), select(enabled, dans_da_new, vals[14]))
y = 1 - a
z = x + y + 1
c = fill(dtype(x), shape(x), 0)
pkm2 = fill(dtype(x), shape(x), 1)
qkm2 = x
pkm1 = x + 1
qkm1 = z * x
ans = pkm1 / qkm1
t = fill(dtype(x), shape(x), 1)
dpkm2_da = fill(dtype(x), shape(x), 0)
dqkm2_da = fill(dtype(x), shape(x), 0)
dpkm1_da = fill(dtype(x), shape(x), 0)
dqkm1_da = -x
dans_da = (dpkm1_da - ans * dqkm1_da) / qkm1
vals = (enabled, ans, t, y, z, c, pkm1, qkm1, pkm2, qkm2, dpkm2_da, dqkm2_da, dpkm1_da, dqkm1_da, dans_da)
vals = _while_helper_func(cond, body, vals)
ans = vals[1]
return ans * ax
def __init__(self,
vocab_size,
embedding_size,
field_size,
param_init='normal',
target='CPU',
slice_mode='batch_slice',
feature_num_list=None,
max_norm=None,
sparse=True,
operator='SUM'):
super(MultiFieldEmbeddingLookup,
self).__init__(vocab_size, embedding_size, param_init, target,
slice_mode, feature_num_list, max_norm, sparse)
self.field_size = validator.check_positive_int(field_size,
'field_size')
self.operator = operator
self.mul = P.Mul()
self.inf_mask_mul = P.Mul()
self.bias_add = P.Add()
self.inf_add = P.Add()
self.merge_op = None
self.count_op = P.UnsortedSegmentSum()
self.abs = P.Abs()
self.equal = P.Equal()
self.add = P.Add()
self.cast = P.Cast()
self.div_no_nan = P.DivNoNan()
self.expand = P.ExpandDims()
self.max_mask_mul = P.Mul()
self.max_no_equal = P.NotEqual()
if operator == MultiFieldEmbeddingLookup.OPERATOR_SUM:
self.merge_op = P.UnsortedSegmentSum()
elif operator == MultiFieldEmbeddingLookup.OPERATOR_MAX:
self.merge_op = P.UnsortedSegmentMax()
elif operator == MultiFieldEmbeddingLookup.OPERATOR_MEAN:
self.merge_op = P.UnsortedSegmentSum()
else:
raise ValueError(
"The operator supports ['SUM', 'MAX', 'MEAN'], but found: " +
str(operator))
parallel_mode = _get_parallel_mode()
is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
ParallelMode.AUTO_PARALLEL)
if slice_mode in ["table_row_slice", "batch_slice"
] and is_auto_parallel:
self.merge_op.shard(
((get_group_size(), 1, 1), (get_group_size(), 1)))
self.expand.shard(((get_group_size(), ), ))
self.bias_add.shard(((1, 1), (1, 1)))
self.mul.shard(
((get_group_size(), 1, 1), (get_group_size(), 1, 1)))
self.count_op.shard(((get_group_size(), 1), (get_group_size(), 1)))
self.add.shard(((get_group_size(), ), (get_group_size(), )))
self.div_no_nan.shard(
((get_group_size(), 1), (get_group_size(), 1)))
self.max_mask_mul.shard(
((get_group_size(), 1), (get_group_size(), 1)))
self.max_no_equal.shard(((1, ), ()))
if operator == MultiFieldEmbeddingLookup.OPERATOR_MAX:
self.equal.shard(((get_group_size(), 1, 1), ()))
self.inf_mask_mul.shard(((get_group_size(), 1, 1), ()))
self.merge_op.shard(
((get_group_size(), 1), (get_group_size(), )))
self.count_op.shard(
((get_group_size(), ), (get_group_size(), )))
self.inf_add.shard(
((get_group_size(), 1, 1), (get_group_size(), 1, 1)))
elif slice_mode == "table_column_slice" and is_auto_parallel:
self.merge_op.shard(((1, 1, get_group_size()), (1, 1)))
self.div_no_nan.shard(((1, get_group_size()), (1, 1)))
self.bias_add.shard(((1, 1), (1, 1)))
self.mul.shard(((1, 1, 1), (1, 1, get_group_size())))
self.count_op.shard(((1, 1), (1, 1)))
self.add.shard(((1, ), (1, )))
self.max_mask_mul.shard(((1, get_group_size()), (1, 1)))
self.expand.shard(((1, ), ))
self.max_no_equal.shard(((1, ), ()))
if operator == MultiFieldEmbeddingLookup.OPERATOR_MAX:
self.equal.shard(((1, 1, 1), ()))
self.inf_mask_mul.shard(((1, 1, 1), ()))
self.merge_op.shard(((1, get_group_size()), (1, )))
self.count_op.shard(((1, ), (1, )))
self.inf_add.shard(((1, 1, get_group_size()), (1, 1, 1)))
else:
if is_auto_parallel:
raise ValueError(
"slice_mode should be ['table_row_slice', 'batch_slice' and \
'table_column_slice'], but get " + str(slice_mode))
# Min value for fp32
self.negative_inf_value = -3.402823466E+38
'desc_inputs': [[4]],
'desc_bprop': [[4]]}),
('RealDiv', {
'block': P.RealDiv(),
'desc_inputs': [[4, 5], [2, 3, 4, 5]],
'desc_bprop': [[2, 3, 4, 5]]}),
('Div', {
'block': P.Div(),
'desc_inputs': [[4, 5], [2, 3, 4, 5]],
'desc_bprop': [[2, 3, 4, 5]]}),
('Equal', {
'block': P.Equal(),
'desc_inputs': [[3, 4, 5], [4, 5]],
'desc_bprop': [Tensor(np.zeros((3, 4, 5), np.bool_))]}),
('NotEqual', {
'block': P.NotEqual(),
'desc_inputs': [[4, 1], [2, 3, 4, 5]],
'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}),
('NotEqual_0', {
'block': P.NotEqual(),
'desc_inputs': [ 1, [2, 3, 4, 5]],
'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))],
'skip': ['backward']}),
('Greater', {
'block': P.Greater(),
'desc_inputs': [[2, 3, 4, 1], [4, 5]],
'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}),
('GreaterEqual', {
'block': P.GreaterEqual(),
'desc_inputs': [[2, 3, 4, 1], [4, 5]],
'desc_bprop': [Tensor(np.ones((2, 3, 4, 5), np.bool_))]}),
# input is not tensor
('EqualCount0', {
'block': (P.EqualCount(), {'exception': TypeError, 'error_keywords': ['EqualCount']}),
'desc_inputs': [5.0, Tensor(np.ones([3, 4]).astype(np.float32))],
'skip': ['backward']}),
# type of x and y not match
('EqualCount1', {
'block': (P.EqualCount(), {'exception': TypeError, 'error_keywords': ['EqualCount']}),
'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32))],
'skip': ['backward']}),
# shape of x and y not match
# input is not tensor
('NotEqual0', {
'block': (P.NotEqual(), {'exception': TypeError, 'error_keywords': ['NotEqual']}),
'desc_inputs': [5.0, Tensor(np.ones([3, 4]).astype(np.float32))],
'skip': ['backward']}),
# type of x and y not match
('NotEqual1', {
'block': (P.NotEqual(), {'exception': TypeError, 'error_keywords': ['NotEqual']}),
'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.int32)), Tensor(np.ones([3, 4]).astype(np.float32))],
'skip': ['backward']}),
# shape of x and y not match
('NotEqual2', {
'block': (P.NotEqual(), {'exception': ValueError, 'error_keywords': ['NotEqual']}),
'desc_inputs': [Tensor(np.ones([3, 4]).astype(np.float32)), Tensor(np.ones([3, 2]).astype(np.float32))],
'skip': ['backward']}),
# input is not tensor
('Greater0', {
('EqualCount1', {
'block': (P.EqualCount(), {
'exception': TypeError,
'error_keywords': ['EqualCount']
}),
'desc_inputs': [
Tensor(np.ones([3, 4]).astype(np.int32)),
Tensor(np.ones([3, 4]).astype(np.float32))
],
'skip': ['backward']
}),
# shape of x and y not match
# shape of x and y not match
('NotEqual2', {
'block': (P.NotEqual(), {
'exception': ValueError,
'error_keywords': ['NotEqual']
}),
'desc_inputs': [
Tensor(np.ones([3, 4]).astype(np.float32)),
Tensor(np.ones([3, 2]).astype(np.float32))
],
'skip': ['backward']
}),
# shape of x and y not match
('Greater2', {
'block': (P.Greater(), {
'exception': ValueError,
'error_keywords': ['Greater']
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().set_strategy(strategy1)
self.notequal = P.NotEqual().set_strategy(strategy2)
def __init__(self, strategy1, strategy2):
super().__init__()
self.matmul = P.MatMul().shard(strategy1)
self.equal = P.Equal().shard(strategy2)
self.notequal = P.NotEqual().shard(strategy2)
self.logical = P.LogicalOr().shard(strategy2)
