def construct(self, x):
"""construct function"""
opt_matmul_0 = self.matmul_0(ops.Cast()(x, dst_type),
ops.Cast()(self.matmul_0_w, dst_type))
opt_add_1 = self.add_1(ops.Cast()(opt_matmul_0, dst_type2),
self.add_1_bias)
return opt_add_1
def construct(self, x):
"""construct function"""
module15_0_opt = self.module15_0(x)
normmodule_0_opt = self.normmodule_0(module15_0_opt)
opt_matmul_0 = self.matmul_0(ops.Cast()(normmodule_0_opt, dst_type),
ops.Cast()(self.matmul_0_w, dst_type))
return ops.Cast()(opt_matmul_0, dst_type2)
def variable_recurrent(self, x, h, seq_length, w_ih, w_hh, b_ih, b_hh):
'''recurrent steps with sequence length'''
time_step = x.shape[0]
h_t = h
if self.is_lstm:
hidden_size = h[0].shape[-1]
zero_output = P.ZerosLike()(h_t[0])
else:
hidden_size = h.shape[-1]
zero_output = P.ZerosLike()(h_t)
seq_length = P.Cast()(seq_length, mindspore.float32)
seq_length = P.BroadcastTo((hidden_size, -1))(seq_length)
seq_length = P.Cast()(seq_length, mindspore.int32)
seq_length = P.Transpose()(seq_length, (1, 0))
outputs = []
state_t = h_t
t = 0
while t < time_step:
x_t = x[t:t + 1:1]
x_t = P.Squeeze(0)(x_t)
h_t = self.cell(x_t, state_t, w_ih, w_hh, b_ih, b_hh)
seq_cond = seq_length > t
if self.is_lstm:
state_t_0 = P.Select()(seq_cond, h_t[0], state_t[0])
state_t_1 = P.Select()(seq_cond, h_t[1], state_t[1])
output = P.Select()(seq_cond, h_t[0], zero_output)
state_t = (state_t_0, state_t_1)
else:
state_t = P.Select()(seq_cond, h_t, state_t)
output = P.Select()(seq_cond, h_t, zero_output)
outputs.append(output)
t += 1
outputs = P.Stack()(outputs)
return outputs, state_t
def construct(self, hidden_state):
"""construct function"""
output = self.matmul(ops.Cast()(hidden_state, dst_type),
ops.Cast()(self.matmul_w, dst_type))
output = self.add(ops.Cast()(output, dst_type2), self.add_bias)
output = self.relu(output)
return output
def construct(self, x):
"""construct function"""
output = self.linear_1(x)
output = self.bert_layer_norm(output)
output = self.matmul(ops.Cast()(output, dst_type),
ops.Cast()(self.matmul_w, dst_type))
return ops.Cast()(output, dst_type2)
def construct(self, state):
"""construct function"""
output = self.linear_1(state)
output = self.bert_layer_norm(output)
output = self.matmul(ops.Cast()(output, dst_type),
ops.Cast()(self.linear_2_weight, dst_type))
output = self.add(ops.Cast()(output, dst_type2), self.linear_2_bias)
return output
def __init__(self, config):
super(CreateAttentionMaskFromInputMask, self).__init__()
self.input_mask = None
self.cast = P.Cast()
self.reshape = P.Reshape()
self.shape = (-1, 1, config.seq_length)
def __init__(self, network, optimizer, sens=1):
super(CenterNetWithLossScaleCell, self).__init__(auto_prefix=False)
self.image = ImagePreProcess()
self.network = network
self.network.set_grad()
self.weights = optimizer.parameters
self.optimizer = optimizer
self.grad = ops.GradOperation(get_by_list=True, sens_param=True)
self.reducer_flag = False
self.allreduce = ops.AllReduce()
self.parallel_mode = context.get_auto_parallel_context("parallel_mode")
if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]:
self.reducer_flag = True
self.grad_reducer = ops.identity
self.degree = 1
if self.reducer_flag:
self.degree = get_group_size()
self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree)
self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
self.cast = ops.Cast()
self.alloc_status = ops.NPUAllocFloatStatus()
self.get_status = ops.NPUGetFloatStatus()
self.clear_before_grad = ops.NPUClearFloatStatus()
self.reduce_sum = ops.ReduceSum(keep_dims=False)
self.base = Tensor(1, mstype.float32)
self.less_equal = ops.LessEqual()
self.grad_scale = GradScale()
self.loss_scale = sens
def __init__(self, bins=10, momentum=0.0, mu=0.02):
super(GHMRLoss, self).__init__()
self.bins = bins
self.momentum = momentum
self.mu = mu
edges_left = np.array([float(x) / bins for x in range(bins)], dtype=np.float32)
self.edges_left = Tensor(edges_left.reshape((bins, 1, 1, 1, 1)))
edges_right = np.array([float(x) / bins for x in range(1, bins + 1)], dtype=np.float32)
edges_right[-1] += 1e-4
self.edges_right = Tensor(edges_right.reshape((bins, 1, 1, 1, 1)))
if momentum >= 0:
self.acc_sum = Parameter(initializer(0, [bins], mstype.float32))
self.abs = ops.Abs()
self.sqrt = ops.Sqrt()
self.cast = ops.Cast()
self.select = ops.Select()
self.reshape = ops.Reshape()
self.reduce_sum = ops.ReduceSum()
self.max = ops.Maximum()
self.less = ops.Less()
self.equal = ops.Equal()
self.greater = ops.Greater()
self.logical_and = ops.LogicalAnd()
self.greater_equal = ops.GreaterEqual()
self.zeros_like = ops.ZerosLike()
self.expand_dims = ops.ExpandDims()
def __init__(self):
super(ImagePreProcess, self).__init__()
self.transpose = ops.Transpose()
self.perm_list = (0, 3, 1, 2)
self.mean = Tensor(data_cfg.mean.reshape((1, 1, 1, 3)))
self.std = Tensor(data_cfg.std.reshape((1, 1, 1, 3)))
self.cast = ops.Cast()
def __init__(self, inc, outc, kernel_size=3, padding=1, stride=1, has_bias=False, modulation=True):
super(DeformConv2d, self).__init__()
self.kernel_size = kernel_size
self.padding = padding
self.stride = stride
self.zero_padding = nn.Pad(((0, 0), (0, 0), (padding, padding), (padding, padding)))
self.conv = nn.Conv2d(inc, outc, kernel_size=kernel_size, pad_mode='valid', padding=0,
stride=kernel_size, has_bias=has_bias)
self.p_conv = nn.Conv2d(inc, 2*kernel_size*kernel_size, kernel_size=self.kernel_size,
pad_mode='pad', padding=self.padding, stride=self.stride)
self.modulation = modulation
if modulation:
self.m_conv = nn.Conv2d(inc, kernel_size*kernel_size, kernel_size=self.kernel_size,
pad_mode='valid', padding=0, stride=self.stride)
if kernel_size % 2 == 0:
raise ValueError("Only odd number is supported, but current kernel sizeis {}".format(kernel_size))
self.N = kernel_size * kernel_size
self.begin = kernel_size // 2
self.sigmoid = ops.Sigmoid()
self.dtype = ops.DType()
self.perm_list = (0, 2, 3, 1)
self.transpose = ops.Transpose()
self.floor = ops.Floor()
self.half = ops.Split(axis=-1, output_num=2)
self.clip_value = ClipByValue()
self.expand_dims = ops.ExpandDims()
self.shape = ops.Shape()
self.cast = ops.Cast()
self._get_offset = GetOffsetPosition(self.begin, self.stride)
self._get_surround = GetSurroundFeature()
self._generate_fm = RegenerateFeatureMap(self.kernel_size)
def __init__(self, config):
super(GetMaskedLMOutput, self).__init__()
self.width = config.hidden_size
self.reshape = ops.Reshape()
self.gather = ops.GatherV2()
weight_init = TruncatedNormal(config.initializer_range)
self.dense = nn.Dense(self.width,
config.hidden_size,
weight_init=weight_init,
activation=config.hidden_act).to_float(config.compute_type)
self.layernorm = nn.LayerNorm((config.hidden_size,)).to_float(config.compute_type)
self.output_bias = Parameter(
initializer(
'zero',
config.vocab_size),
name='output_bias')
self.matmul = ops.MatMul(transpose_b=True)
self.log_softmax = nn.LogSoftmax(axis=-1)
self.shape_flat_offsets = (-1, 1)
self.rng = Tensor(np.array(range(0, config.batch_size)).astype(np.int32))
self.last_idx = (-1,)
self.shape_flat_sequence_tensor = (config.batch_size * config.seq_length, self.width)
self.seq_length_tensor = Tensor(np.array((config.seq_length,)).astype(np.int32))
self.cast = ops.Cast()
self.compute_type = config.compute_type
self.dtype = config.dtype
def __init__(self, network, optimizer, scale_update_cell=None):
super(BertTrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False)
self.network = network
self.weights = ParameterTuple(network.trainable_params())
self.optimizer = optimizer
self.grad = ops.GradOperation(
get_by_list=True,
sens_param=True)
self.reducer_flag = False
self.allreduce = ops.AllReduce()
self.parallel_mode = context.get_auto_parallel_context("parallel_mode")
if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]:
self.reducer_flag = True
self.grad_reducer = ops.identity
self.degree = 1
if self.reducer_flag:
self.degree = get_group_size()
self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree)
self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
self.cast = ops.Cast()
self.alloc_status = ops.NPUAllocFloatStatus()
self.get_status = ops.NPUGetFloatStatus()
self.clear_before_grad = ops.NPUClearFloatStatus()
self.reduce_sum = ops.ReduceSum(keep_dims=False)
self.depend_parameter_use = ops.ControlDepend(depend_mode=1)
self.base = Tensor(1, mstype.float32)
self.less_equal = ops.LessEqual()
self.hyper_map = ops.HyperMap()
self.loss_scale = None
self.loss_scaling_manager = scale_update_cell
if scale_update_cell:
self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32),
name="loss_scale")
def __init__(self, config):
super(GetNextSentenceOutput, self).__init__()
self.log_softmax = _selected_ops.LogSoftmax()
weight_init = TruncatedNormal(config.initializer_range)
self.dense = nn.Dense(config.hidden_size, 2,
weight_init=weight_init, has_bias=True).to_float(config.compute_type)
self.dtype = config.dtype
self.cast = ops.Cast()
def __init__(self, kernel=3, enable_nms_fp16=True):
super(NMS, self).__init__()
self.pad = (kernel - 1) // 2
self.cast = ops.Cast()
self.dtype = ops.DType()
self.equal = ops.Equal()
self.max_pool = nn.MaxPool2d(kernel, stride=1, pad_mode="same")
self.enable_fp16 = enable_nms_fp16
def __init__(self):
super(GatherTopKChannel, self).__init__()
self.shape = ops.Shape()
self.reshape = ops.Reshape()
self.topk = ops.TopK(sorted=True)
self.cast = ops.Cast()
self.dtype = ops.DType()
self.mod = ops.Mod()
self.div = ops.Div()
def __init__(self, src_type=mstype.float32, dst_type=mstype.float32):
super(SaturateCast, self).__init__()
np_type = mstype.dtype_to_nptype(dst_type)
self.tensor_min_type = float(np.finfo(np_type).min)
self.tensor_max_type = float(np.finfo(np_type).max)
self.min_op = P.Minimum()
self.max_op = P.Maximum()
self.cast = P.Cast()
self.dst_type = dst_type
def __init__(self, learning_rate, warmup_steps, multi_epochs, steps_per_epoch, factor=10):
super(CenterNetMultiEpochsDecayLR, self).__init__()
self.warmup_flag = False
if warmup_steps > 0:
self.warmup_flag = True
self.warmup_lr = WarmUpLR(learning_rate, warmup_steps)
self.decay_lr = MultiEpochsDecayLR(learning_rate, multi_epochs, steps_per_epoch, factor)
self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32))
self.greater = ops.Greater()
self.one = Tensor(np.array([1.0]).astype(np.float32))
self.cast = ops.Cast()
def __init__(self, length, max_relative_position):
super(RelaPosMatrixGenerator, self).__init__()
self._length = length
self._max_relative_position = Tensor(max_relative_position, dtype=mstype.int32)
self._min_relative_position = Tensor(-max_relative_position, dtype=mstype.int32)
self.range_length = -length + 1
self.tile = ops.Tile()
self.range_mat = ops.Reshape()
self.sub = ops.Sub()
self.expanddims = ops.ExpandDims()
self.cast = ops.Cast()
def __init__(self, config):
super(BertPretrainingLoss, self).__init__()
self.vocab_size = config.vocab_size
self.onehot = ops.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.reduce_sum = ops.ReduceSum()
self.reduce_mean = ops.ReduceMean()
self.reshape = ops.Reshape()
self.last_idx = (-1,)
self.neg = ops.Neg()
self.cast = ops.Cast()
def __init__(self, length, max_relative_position):
super(RelaPosMatrixGenerator, self).__init__()
self._length = length
self._max_relative_position = max_relative_position
self._min_relative_position = -max_relative_position
self.range_length = -length + 1
self.tile = P.Tile()
self.range_mat = P.Reshape()
self.sub = P.Sub()
self.expanddims = P.ExpandDims()
self.cast = P.Cast()
def __init__(self, learning_rate, end_learning_rate, warmup_steps, decay_steps, power):
super(CenterNetPolynomialDecayLR, self).__init__()
self.warmup_flag = False
if warmup_steps > 0:
self.warmup_flag = True
self.warmup_lr = WarmUpLR(learning_rate, warmup_steps)
self.decay_lr = PolynomialDecayLR(learning_rate, end_learning_rate, decay_steps, power)
self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32))
self.greater = ops.Greater()
self.one = Tensor(np.array([1.0]).astype(np.float32))
self.cast = ops.Cast()
def construct(self, input_ids, attn_mask, token_type_ids, context_mask,
square_mask, packing_mask, cache_mask, para_start_mapping,
sent_end_mapping):
"""construct function"""
state = self.encoder(attn_mask, input_ids, token_type_ids)
para_state = self.bmm(ops.Cast()(para_start_mapping, dst_type),
ops.Cast()(state, dst_type)) # [B, 2, D]
sent_state = self.bmm(ops.Cast()(sent_end_mapping, dst_type),
ops.Cast()(state, dst_type)) # [B, max_sent, D]
q_type, start, end, para_logit, sent_logit = self.downstream(
ops.Cast()(para_state, dst_type2),
ops.Cast()(sent_state, dst_type2), state, context_mask)
outer = start[:, :, None] + end[:, None]
outer_mask = cache_mask
outer_mask = square_mask * outer_mask[None]
outer = outer - 1e30 * (1 - outer_mask)
outer = outer - 1e30 * packing_mask[:, :, None]
max_row = ops.ReduceMax()(outer, 2)
y1 = ops.Argmax()(max_row)
max_col = ops.ReduceMax()(outer, 1)
y2 = ops.Argmax()(max_col)
return start, end, q_type, para_logit, sent_logit, y1, y2
def __init__(self, begin, stride):
super(GetOffsetPosition, self).__init__()
self.begin = begin
self.stride = stride
self.meshgrid = ops.Meshgrid()
self.shape = ops.Shape()
self.reshape = ops.Reshape()
self.cat_a0 = ops.Concat(axis=0)
self.cat_a1 = ops.Concat(axis=1)
self.tile = ops.Tile()
self.dtype = ops.DType()
self.range = nn.Range(-self.begin, self.begin + 1)
self.cast = ops.Cast()
def __init__(self, src_type=mstype.float32, dst_type=mstype.float32):
super(SaturateCast, self).__init__()
np_type = mstype.dtype_to_nptype(dst_type)
min_type = np.finfo(np_type).min
max_type = np.finfo(np_type).max
self.tensor_min_type = Tensor([min_type], dtype=src_type)
self.tensor_max_type = Tensor([max_type], dtype=src_type)
self.min_op = ops.Minimum()
self.max_op = ops.Maximum()
self.cast = ops.Cast()
self.dst_type = dst_type
def __init__(self, learning_rate, multi_epochs, steps_per_epoch, factor=10):
super(MultiEpochsDecayLR, self).__init__()
if not isinstance(multi_epochs, (list, tuple)):
raise TypeError("multi_epochs must be list or tuple.")
self.multi_epochs = Tensor(np.array(multi_epochs, dtype=np.float32) * steps_per_epoch)
self.num = len(multi_epochs)
self.start_learning_rate = learning_rate
self.steps_per_epoch = steps_per_epoch
self.factor = factor
self.pow = ops.Pow()
self.cast = ops.Cast()
self.less_equal = ops.LessEqual()
self.reduce_sum = ops.ReduceSum()
def __init__(self, mode='l1'):
super(RegLoss, self).__init__()
self.reduce_sum = ops.ReduceSum()
self.cast = ops.Cast()
self.expand_dims = ops.ExpandDims()
self.reshape = ops.Reshape()
self.gather_feature = TransposeGatherFeature()
if mode == 'l1':
self.loss = nn.L1Loss(reduction='sum')
elif mode == 'sl1':
self.loss = nn.SmoothL1Loss()
else:
self.loss = None
def __init__(self,
in_channels,
out_channels,
initializer_range=0.02,
dropout_prob=0.1,
compute_type=mstype.float32):
super(BertOutput, self).__init__()
self.dense = nn.Dense(in_channels, out_channels,
weight_init=TruncatedNormal(initializer_range)).to_float(compute_type)
self.dropout = nn.Dropout(1 - dropout_prob)
self.dropout_prob = dropout_prob
self.add = P.Add()
self.layernorm = nn.LayerNorm((out_channels,), epsilon=1e-5).to_float(compute_type)
self.cast = P.Cast()
def __init__(self, alpha=2, beta=4):
super(FocalLoss, self).__init__()
self.alpha = alpha
self.beta = beta
self.pow = ops.Pow()
self.log = ops.Log()
self.select = ops.Select()
self.equal = ops.Equal()
self.less = ops.Less()
self.cast = ops.Cast()
self.fill = ops.Fill()
self.dtype = ops.DType()
self.shape = ops.Shape()
self.reduce_sum = ops.ReduceSum()
def __init__(self, config):
super(CreateAttentionMaskFromInputMask, self).__init__()
self.input_mask_from_dataset = config.input_mask_from_dataset
self.input_mask = None
if not self.input_mask_from_dataset:
self.input_mask = initializer(
"ones", [config.batch_size, config.seq_length], mstype.int32).to_tensor()
self.cast = ops.Cast()
self.reshape = ops.Reshape()
self.shape = (config.batch_size, 1, config.seq_length)
self.broadcast_ones = initializer(
"ones", [config.batch_size, config.seq_length, 1], mstype.float32).to_tensor()
self.batch_matmul = ops.BatchMatMul()
