def construct(self, x): """construct function""" opt_matmul_0 = self.matmul_0(ops.Cast()(x, dst_type), ops.Cast()(self.matmul_0_w, dst_type)) opt_add_1 = self.add_1(ops.Cast()(opt_matmul_0, dst_type2), self.add_1_bias) return opt_add_1
def construct(self, x): """construct function""" module15_0_opt = self.module15_0(x) normmodule_0_opt = self.normmodule_0(module15_0_opt) opt_matmul_0 = self.matmul_0(ops.Cast()(normmodule_0_opt, dst_type), ops.Cast()(self.matmul_0_w, dst_type)) return ops.Cast()(opt_matmul_0, dst_type2)
def variable_recurrent(self, x, h, seq_length, w_ih, w_hh, b_ih, b_hh): '''recurrent steps with sequence length''' time_step = x.shape[0] h_t = h if self.is_lstm: hidden_size = h[0].shape[-1] zero_output = P.ZerosLike()(h_t[0]) else: hidden_size = h.shape[-1] zero_output = P.ZerosLike()(h_t) seq_length = P.Cast()(seq_length, mindspore.float32) seq_length = P.BroadcastTo((hidden_size, -1))(seq_length) seq_length = P.Cast()(seq_length, mindspore.int32) seq_length = P.Transpose()(seq_length, (1, 0)) outputs = [] state_t = h_t t = 0 while t < time_step: x_t = x[t:t + 1:1] x_t = P.Squeeze(0)(x_t) h_t = self.cell(x_t, state_t, w_ih, w_hh, b_ih, b_hh) seq_cond = seq_length > t if self.is_lstm: state_t_0 = P.Select()(seq_cond, h_t[0], state_t[0]) state_t_1 = P.Select()(seq_cond, h_t[1], state_t[1]) output = P.Select()(seq_cond, h_t[0], zero_output) state_t = (state_t_0, state_t_1) else: state_t = P.Select()(seq_cond, h_t, state_t) output = P.Select()(seq_cond, h_t, zero_output) outputs.append(output) t += 1 outputs = P.Stack()(outputs) return outputs, state_t
def construct(self, hidden_state): """construct function""" output = self.matmul(ops.Cast()(hidden_state, dst_type), ops.Cast()(self.matmul_w, dst_type)) output = self.add(ops.Cast()(output, dst_type2), self.add_bias) output = self.relu(output) return output
def construct(self, x): """construct function""" output = self.linear_1(x) output = self.bert_layer_norm(output) output = self.matmul(ops.Cast()(output, dst_type), ops.Cast()(self.matmul_w, dst_type)) return ops.Cast()(output, dst_type2)
def construct(self, state): """construct function""" output = self.linear_1(state) output = self.bert_layer_norm(output) output = self.matmul(ops.Cast()(output, dst_type), ops.Cast()(self.linear_2_weight, dst_type)) output = self.add(ops.Cast()(output, dst_type2), self.linear_2_bias) return output
def __init__(self, config): super(CreateAttentionMaskFromInputMask, self).__init__() self.input_mask = None self.cast = P.Cast() self.reshape = P.Reshape() self.shape = (-1, 1, config.seq_length)
def __init__(self, network, optimizer, sens=1): super(CenterNetWithLossScaleCell, self).__init__(auto_prefix=False) self.image = ImagePreProcess() self.network = network self.network.set_grad() self.weights = optimizer.parameters self.optimizer = optimizer self.grad = ops.GradOperation(get_by_list=True, sens_param=True) self.reducer_flag = False self.allreduce = ops.AllReduce() self.parallel_mode = context.get_auto_parallel_context("parallel_mode") if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]: self.reducer_flag = True self.grad_reducer = ops.identity self.degree = 1 if self.reducer_flag: self.degree = get_group_size() self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree) self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) self.cast = ops.Cast() self.alloc_status = ops.NPUAllocFloatStatus() self.get_status = ops.NPUGetFloatStatus() self.clear_before_grad = ops.NPUClearFloatStatus() self.reduce_sum = ops.ReduceSum(keep_dims=False) self.base = Tensor(1, mstype.float32) self.less_equal = ops.LessEqual() self.grad_scale = GradScale() self.loss_scale = sens
def __init__(self, bins=10, momentum=0.0, mu=0.02): super(GHMRLoss, self).__init__() self.bins = bins self.momentum = momentum self.mu = mu edges_left = np.array([float(x) / bins for x in range(bins)], dtype=np.float32) self.edges_left = Tensor(edges_left.reshape((bins, 1, 1, 1, 1))) edges_right = np.array([float(x) / bins for x in range(1, bins + 1)], dtype=np.float32) edges_right[-1] += 1e-4 self.edges_right = Tensor(edges_right.reshape((bins, 1, 1, 1, 1))) if momentum >= 0: self.acc_sum = Parameter(initializer(0, [bins], mstype.float32)) self.abs = ops.Abs() self.sqrt = ops.Sqrt() self.cast = ops.Cast() self.select = ops.Select() self.reshape = ops.Reshape() self.reduce_sum = ops.ReduceSum() self.max = ops.Maximum() self.less = ops.Less() self.equal = ops.Equal() self.greater = ops.Greater() self.logical_and = ops.LogicalAnd() self.greater_equal = ops.GreaterEqual() self.zeros_like = ops.ZerosLike() self.expand_dims = ops.ExpandDims()
def __init__(self): super(ImagePreProcess, self).__init__() self.transpose = ops.Transpose() self.perm_list = (0, 3, 1, 2) self.mean = Tensor(data_cfg.mean.reshape((1, 1, 1, 3))) self.std = Tensor(data_cfg.std.reshape((1, 1, 1, 3))) self.cast = ops.Cast()
def __init__(self, inc, outc, kernel_size=3, padding=1, stride=1, has_bias=False, modulation=True): super(DeformConv2d, self).__init__() self.kernel_size = kernel_size self.padding = padding self.stride = stride self.zero_padding = nn.Pad(((0, 0), (0, 0), (padding, padding), (padding, padding))) self.conv = nn.Conv2d(inc, outc, kernel_size=kernel_size, pad_mode='valid', padding=0, stride=kernel_size, has_bias=has_bias) self.p_conv = nn.Conv2d(inc, 2*kernel_size*kernel_size, kernel_size=self.kernel_size, pad_mode='pad', padding=self.padding, stride=self.stride) self.modulation = modulation if modulation: self.m_conv = nn.Conv2d(inc, kernel_size*kernel_size, kernel_size=self.kernel_size, pad_mode='valid', padding=0, stride=self.stride) if kernel_size % 2 == 0: raise ValueError("Only odd number is supported, but current kernel sizeis {}".format(kernel_size)) self.N = kernel_size * kernel_size self.begin = kernel_size // 2 self.sigmoid = ops.Sigmoid() self.dtype = ops.DType() self.perm_list = (0, 2, 3, 1) self.transpose = ops.Transpose() self.floor = ops.Floor() self.half = ops.Split(axis=-1, output_num=2) self.clip_value = ClipByValue() self.expand_dims = ops.ExpandDims() self.shape = ops.Shape() self.cast = ops.Cast() self._get_offset = GetOffsetPosition(self.begin, self.stride) self._get_surround = GetSurroundFeature() self._generate_fm = RegenerateFeatureMap(self.kernel_size)
def __init__(self, config): super(GetMaskedLMOutput, self).__init__() self.width = config.hidden_size self.reshape = ops.Reshape() self.gather = ops.GatherV2() weight_init = TruncatedNormal(config.initializer_range) self.dense = nn.Dense(self.width, config.hidden_size, weight_init=weight_init, activation=config.hidden_act).to_float(config.compute_type) self.layernorm = nn.LayerNorm((config.hidden_size,)).to_float(config.compute_type) self.output_bias = Parameter( initializer( 'zero', config.vocab_size), name='output_bias') self.matmul = ops.MatMul(transpose_b=True) self.log_softmax = nn.LogSoftmax(axis=-1) self.shape_flat_offsets = (-1, 1) self.rng = Tensor(np.array(range(0, config.batch_size)).astype(np.int32)) self.last_idx = (-1,) self.shape_flat_sequence_tensor = (config.batch_size * config.seq_length, self.width) self.seq_length_tensor = Tensor(np.array((config.seq_length,)).astype(np.int32)) self.cast = ops.Cast() self.compute_type = config.compute_type self.dtype = config.dtype
def __init__(self, network, optimizer, scale_update_cell=None): super(BertTrainOneStepWithLossScaleCell, self).__init__(auto_prefix=False) self.network = network self.weights = ParameterTuple(network.trainable_params()) self.optimizer = optimizer self.grad = ops.GradOperation( get_by_list=True, sens_param=True) self.reducer_flag = False self.allreduce = ops.AllReduce() self.parallel_mode = context.get_auto_parallel_context("parallel_mode") if self.parallel_mode in [ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL]: self.reducer_flag = True self.grad_reducer = ops.identity self.degree = 1 if self.reducer_flag: self.degree = get_group_size() self.grad_reducer = DistributedGradReducer(optimizer.parameters, False, self.degree) self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) self.cast = ops.Cast() self.alloc_status = ops.NPUAllocFloatStatus() self.get_status = ops.NPUGetFloatStatus() self.clear_before_grad = ops.NPUClearFloatStatus() self.reduce_sum = ops.ReduceSum(keep_dims=False) self.depend_parameter_use = ops.ControlDepend(depend_mode=1) self.base = Tensor(1, mstype.float32) self.less_equal = ops.LessEqual() self.hyper_map = ops.HyperMap() self.loss_scale = None self.loss_scaling_manager = scale_update_cell if scale_update_cell: self.loss_scale = Parameter(Tensor(scale_update_cell.get_loss_scale(), dtype=mstype.float32), name="loss_scale")
def __init__(self, config): super(GetNextSentenceOutput, self).__init__() self.log_softmax = _selected_ops.LogSoftmax() weight_init = TruncatedNormal(config.initializer_range) self.dense = nn.Dense(config.hidden_size, 2, weight_init=weight_init, has_bias=True).to_float(config.compute_type) self.dtype = config.dtype self.cast = ops.Cast()
def __init__(self, kernel=3, enable_nms_fp16=True): super(NMS, self).__init__() self.pad = (kernel - 1) // 2 self.cast = ops.Cast() self.dtype = ops.DType() self.equal = ops.Equal() self.max_pool = nn.MaxPool2d(kernel, stride=1, pad_mode="same") self.enable_fp16 = enable_nms_fp16
def __init__(self): super(GatherTopKChannel, self).__init__() self.shape = ops.Shape() self.reshape = ops.Reshape() self.topk = ops.TopK(sorted=True) self.cast = ops.Cast() self.dtype = ops.DType() self.mod = ops.Mod() self.div = ops.Div()
def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() np_type = mstype.dtype_to_nptype(dst_type) self.tensor_min_type = float(np.finfo(np_type).min) self.tensor_max_type = float(np.finfo(np_type).max) self.min_op = P.Minimum() self.max_op = P.Maximum() self.cast = P.Cast() self.dst_type = dst_type
def __init__(self, learning_rate, warmup_steps, multi_epochs, steps_per_epoch, factor=10): super(CenterNetMultiEpochsDecayLR, self).__init__() self.warmup_flag = False if warmup_steps > 0: self.warmup_flag = True self.warmup_lr = WarmUpLR(learning_rate, warmup_steps) self.decay_lr = MultiEpochsDecayLR(learning_rate, multi_epochs, steps_per_epoch, factor) self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32)) self.greater = ops.Greater() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.cast = ops.Cast()
def __init__(self, length, max_relative_position): super(RelaPosMatrixGenerator, self).__init__() self._length = length self._max_relative_position = Tensor(max_relative_position, dtype=mstype.int32) self._min_relative_position = Tensor(-max_relative_position, dtype=mstype.int32) self.range_length = -length + 1 self.tile = ops.Tile() self.range_mat = ops.Reshape() self.sub = ops.Sub() self.expanddims = ops.ExpandDims() self.cast = ops.Cast()
def __init__(self, config): super(BertPretrainingLoss, self).__init__() self.vocab_size = config.vocab_size self.onehot = ops.OneHot() self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.reduce_sum = ops.ReduceSum() self.reduce_mean = ops.ReduceMean() self.reshape = ops.Reshape() self.last_idx = (-1,) self.neg = ops.Neg() self.cast = ops.Cast()
def __init__(self, length, max_relative_position): super(RelaPosMatrixGenerator, self).__init__() self._length = length self._max_relative_position = max_relative_position self._min_relative_position = -max_relative_position self.range_length = -length + 1 self.tile = P.Tile() self.range_mat = P.Reshape() self.sub = P.Sub() self.expanddims = P.ExpandDims() self.cast = P.Cast()
def __init__(self, learning_rate, end_learning_rate, warmup_steps, decay_steps, power): super(CenterNetPolynomialDecayLR, self).__init__() self.warmup_flag = False if warmup_steps > 0: self.warmup_flag = True self.warmup_lr = WarmUpLR(learning_rate, warmup_steps) self.decay_lr = PolynomialDecayLR(learning_rate, end_learning_rate, decay_steps, power) self.warmup_steps = Tensor(np.array([warmup_steps]).astype(np.float32)) self.greater = ops.Greater() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.cast = ops.Cast()
def construct(self, input_ids, attn_mask, token_type_ids, context_mask, square_mask, packing_mask, cache_mask, para_start_mapping, sent_end_mapping): """construct function""" state = self.encoder(attn_mask, input_ids, token_type_ids) para_state = self.bmm(ops.Cast()(para_start_mapping, dst_type), ops.Cast()(state, dst_type)) # [B, 2, D] sent_state = self.bmm(ops.Cast()(sent_end_mapping, dst_type), ops.Cast()(state, dst_type)) # [B, max_sent, D] q_type, start, end, para_logit, sent_logit = self.downstream( ops.Cast()(para_state, dst_type2), ops.Cast()(sent_state, dst_type2), state, context_mask) outer = start[:, :, None] + end[:, None] outer_mask = cache_mask outer_mask = square_mask * outer_mask[None] outer = outer - 1e30 * (1 - outer_mask) outer = outer - 1e30 * packing_mask[:, :, None] max_row = ops.ReduceMax()(outer, 2) y1 = ops.Argmax()(max_row) max_col = ops.ReduceMax()(outer, 1) y2 = ops.Argmax()(max_col) return start, end, q_type, para_logit, sent_logit, y1, y2
def __init__(self, begin, stride): super(GetOffsetPosition, self).__init__() self.begin = begin self.stride = stride self.meshgrid = ops.Meshgrid() self.shape = ops.Shape() self.reshape = ops.Reshape() self.cat_a0 = ops.Concat(axis=0) self.cat_a1 = ops.Concat(axis=1) self.tile = ops.Tile() self.dtype = ops.DType() self.range = nn.Range(-self.begin, self.begin + 1) self.cast = ops.Cast()
def __init__(self, src_type=mstype.float32, dst_type=mstype.float32): super(SaturateCast, self).__init__() np_type = mstype.dtype_to_nptype(dst_type) min_type = np.finfo(np_type).min max_type = np.finfo(np_type).max self.tensor_min_type = Tensor([min_type], dtype=src_type) self.tensor_max_type = Tensor([max_type], dtype=src_type) self.min_op = ops.Minimum() self.max_op = ops.Maximum() self.cast = ops.Cast() self.dst_type = dst_type
def __init__(self, learning_rate, multi_epochs, steps_per_epoch, factor=10): super(MultiEpochsDecayLR, self).__init__() if not isinstance(multi_epochs, (list, tuple)): raise TypeError("multi_epochs must be list or tuple.") self.multi_epochs = Tensor(np.array(multi_epochs, dtype=np.float32) * steps_per_epoch) self.num = len(multi_epochs) self.start_learning_rate = learning_rate self.steps_per_epoch = steps_per_epoch self.factor = factor self.pow = ops.Pow() self.cast = ops.Cast() self.less_equal = ops.LessEqual() self.reduce_sum = ops.ReduceSum()
def __init__(self, mode='l1'): super(RegLoss, self).__init__() self.reduce_sum = ops.ReduceSum() self.cast = ops.Cast() self.expand_dims = ops.ExpandDims() self.reshape = ops.Reshape() self.gather_feature = TransposeGatherFeature() if mode == 'l1': self.loss = nn.L1Loss(reduction='sum') elif mode == 'sl1': self.loss = nn.SmoothL1Loss() else: self.loss = None
def __init__(self, in_channels, out_channels, initializer_range=0.02, dropout_prob=0.1, compute_type=mstype.float32): super(BertOutput, self).__init__() self.dense = nn.Dense(in_channels, out_channels, weight_init=TruncatedNormal(initializer_range)).to_float(compute_type) self.dropout = nn.Dropout(1 - dropout_prob) self.dropout_prob = dropout_prob self.add = P.Add() self.layernorm = nn.LayerNorm((out_channels,), epsilon=1e-5).to_float(compute_type) self.cast = P.Cast()
def __init__(self, alpha=2, beta=4): super(FocalLoss, self).__init__() self.alpha = alpha self.beta = beta self.pow = ops.Pow() self.log = ops.Log() self.select = ops.Select() self.equal = ops.Equal() self.less = ops.Less() self.cast = ops.Cast() self.fill = ops.Fill() self.dtype = ops.DType() self.shape = ops.Shape() self.reduce_sum = ops.ReduceSum()
def __init__(self, config): super(CreateAttentionMaskFromInputMask, self).__init__() self.input_mask_from_dataset = config.input_mask_from_dataset self.input_mask = None if not self.input_mask_from_dataset: self.input_mask = initializer( "ones", [config.batch_size, config.seq_length], mstype.int32).to_tensor() self.cast = ops.Cast() self.reshape = ops.Reshape() self.shape = (config.batch_size, 1, config.seq_length) self.broadcast_ones = initializer( "ones", [config.batch_size, config.seq_length, 1], mstype.float32).to_tensor() self.batch_matmul = ops.BatchMatMul()