def rnn_relu_cell(input, hidden, w_ih, w_hh, b_ih, b_hh): if b_ih is None: igates = P.MatMul(False, True)(input, w_ih) hgates = P.MatMul(False, True)(hidden, w_hh) else: igates = P.MatMul(False, True)(input, w_ih) + b_ih hgates = P.MatMul(False, True)(hidden, w_hh) + b_hh return P.ReLU()(igates + hgates)
def rnn_relu_cell(inputs, hidden, w_ih, w_hh, b_ih, b_hh): '''RNN cell function with relu activation''' if b_ih is None: igates = P.MatMul(False, True)(inputs, w_ih) hgates = P.MatMul(False, True)(hidden, w_hh) else: igates = P.MatMul(False, True)(inputs, w_ih) + b_ih hgates = P.MatMul(False, True)(hidden, w_hh) + b_hh return P.ReLU()(igates + hgates)
def __init__(self, config): super(GetMaskedLMOutput, self).__init__() self.width = config.hidden_size self.reshape = ops.Reshape() self.gather = ops.GatherV2() weight_init = TruncatedNormal(config.initializer_range) self.dense = nn.Dense(self.width, config.hidden_size, weight_init=weight_init, activation=config.hidden_act).to_float(config.compute_type) self.layernorm = nn.LayerNorm((config.hidden_size,)).to_float(config.compute_type) self.output_bias = Parameter( initializer( 'zero', config.vocab_size), name='output_bias') self.matmul = ops.MatMul(transpose_b=True) self.log_softmax = nn.LogSoftmax(axis=-1) self.shape_flat_offsets = (-1, 1) self.rng = Tensor(np.array(range(0, config.batch_size)).astype(np.int32)) self.last_idx = (-1,) self.shape_flat_sequence_tensor = (config.batch_size * config.seq_length, self.width) self.seq_length_tensor = Tensor(np.array((config.seq_length,)).astype(np.int32)) self.cast = ops.Cast() self.compute_type = config.compute_type self.dtype = config.dtype
def __init__(self, input_size: int, hidden_size: int, bias: bool = True, nonlinearity: str = 'tanh'): super().__init__() self.input_size = input_size self.hidden_size = hidden_size self.bias = bias stdv = 1 / math.sqrt(hidden_size) self.weight_ih = Parameter( Tensor( np.random.uniform(-stdv, stdv, (input_size, hidden_size)).astype( np.float32))) self.weight_hh = Parameter( Tensor( np.random.uniform(-stdv, stdv, (hidden_size, hidden_size)).astype( np.float32))) if bias: self.bias_ih = Parameter( Tensor( np.random.uniform(-stdv, stdv, (hidden_size)).astype(np.float32))) self.bias_hh = Parameter( Tensor( np.random.uniform(-stdv, stdv, (hidden_size)).astype(np.float32))) self.nonlinearity = self.nonlinearity_dict[nonlinearity] self.mm = P.MatMul()
def gru_cell(input, hidden, w_ih, w_hh, b_ih, b_hh): if b_ih is None: gi = P.MatMul(False, True)(input, w_ih) gh = P.MatMul(False, True)(hidden, w_hh) else: gi = P.MatMul(False, True)(input, w_ih) + b_ih gh = P.MatMul(False, True)(hidden, w_hh) + b_hh i_r, i_i, i_n = P.Split(1, 3)(gi) h_r, h_i, h_n = P.Split(1, 3)(gh) resetgate = P.Sigmoid()(i_r + h_r) inputgate = P.Sigmoid()(i_i + h_i) newgate = P.Tanh()(i_n + resetgate * h_n) hy = newgate + inputgate * (hidden - newgate) return hy
def lstm_cell(input, hidden, w_ih, w_hh, b_ih, b_hh): hx, cx = hidden if b_ih is None: gates = P.MatMul(False, True)(input, w_ih) + P.MatMul(False, True)(hx, w_hh) else: gates = P.MatMul(False, True)(input, w_ih) + P.MatMul(False, True)(hx, w_hh) + b_ih + b_hh ingate, forgetgate, cellgate, outgate = P.Split(1, 4)(gates) ingate = P.Sigmoid()(ingate) forgetgate = P.Sigmoid()(forgetgate) cellgate = P.Tanh()(cellgate) outgate = P.Sigmoid()(outgate) cy = (forgetgate * cx) + (ingate * cellgate) hy = outgate * P.Tanh()(cy) return hy, cy
def __init__(self, matmul_size, transpose_a=False, transpose_b=False, strategy=None): """init""" super().__init__() matmul_np = np.full(matmul_size, 0.5, dtype=np.float32) self.matmul_weight = Parameter(Tensor(matmul_np)) self.matmul = ops.MatMul(transpose_a=transpose_a, transpose_b=transpose_b) self.neg = ops.Neg() if strategy is not None: self.matmul.shard(strategy)
def __init__(self, in1_channels, in2_channels, out_channels, weight_init=None, bias_init=None, has_bias=True): super().__init__() # self.in_channels = Validator.check_positive_int(in1_channels, "in1_channels", self.cls_name) # self.in_channels = Validator.check_positive_int(in2_channels, "in2_channels", self.cls_name) # self.out_channels = Validator.check_positive_int(out_channels, "out_channels", self.cls_name) # self.has_bias = Validator.check_bool(has_bias, "has_bias", self.cls_name) self.in1_channels = in1_channels self.in2_channels = in2_channels self.out_channels = out_channels self.has_bias = has_bias bound = 1 / math.sqrt(in1_channels) if weight_init is None: weight_init = Uniform(bound) if isinstance(weight_init, Tensor): if weight_init.ndim != 3 or weight_init.shape[0] != out_channels or \ weight_init.shape[1] != in1_channels or weight_init.shape[2] != in2_channels: raise ValueError(f"For '{self.cls_name}', weight init shape error. The ndim of 'weight_init' must " f"be equal to 3, the first dim must be equal to 'out_channels', the " f"second dim must be equal to 'in1_channels', and the third dim must be " f"equal to 'in2_channels'. But got 'weight_init': {weight_init}, " f"'out_channels': {out_channels}, 'in_channels': {in1_channels}, " f"'in2_channels': {in2_channels}") self.weight = Parameter(initializer(weight_init, (out_channels, in1_channels, in2_channels)), 'weight') if self.has_bias: if bias_init is None: bias_init = Uniform(bound) if isinstance(bias_init, Tensor): if bias_init.ndim != 1 or bias_init.shape[0] != out_channels: raise ValueError(f"For '{self.cls_name}', bias init shape error. The ndim of 'bias_init' should " f"be equal to 1, and the first dim must be equal to 'out_channels'. But got " f"'bias_init': {bias_init}, 'out_channels': {out_channels}.") self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") self.bias_add = P.BiasAdd() self.matmul = P.MatMul()
def __init__(self, input_size: int, hidden_size: int, bias: bool = True, \ cell_clip=None, proj_size=None, proj_clip=None): super().__init__(input_size, hidden_size, bias, num_chunks=4, proj_size=proj_size) self.cell_clip = cell_clip self.proj_size = proj_size self.proj_clip = proj_clip if proj_size is not None: self.proj_weight = Parameter( Tensor( np.random.randn(hidden_size, proj_size).astype(np.float32))) self.matmul = P.MatMul() self.cell_type = 'LSTM'
def __init__(self, vocab_size, embedding_size, embedding_shape, use_one_hot_embeddings=False, initializer_range=0.02): super(EmbeddingLookup, self).__init__() self.vocab_size = vocab_size self.use_one_hot_embeddings = use_one_hot_embeddings self.embedding_table = Parameter(initializer (TruncatedNormal(initializer_range), [vocab_size, embedding_size])) self.expand = P.ExpandDims() self.shape_flat = (-1,) self.gather = P.Gather() self.one_hot = P.OneHot() self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.0, mstype.float32) self.array_mul = P.MatMul() self.reshape = P.Reshape() self.shape = tuple(embedding_shape)
def __init__(self, config, is_training, num_tokens, dropout_prob=0.0, use_one_hot_embeddings=False): super(BertPoetryModel, self).__init__() self.bert = BertModel(config, is_training, use_one_hot_embeddings) self.num_tokens = num_tokens idx = np.arange(config.seq_length) mask = idx[None, :] <= idx[:, None] self.mask = Tensor([mask], mstype.float32) self.MLM_Dense = nn.Dense(config.hidden_size, config.hidden_size,\ has_bias=True, weight_init=TruncatedNormal(0.02),\ activation='gelu').to_float(mstype.float16) self.layer_norm = nn.LayerNorm((config.hidden_size,)) self.matmul = ops.MatMul(transpose_b=True) self.biasadd = Parameter(initializer('zero', self.num_tokens), name='MLM_output_biasadd') self.softmax = ops.Softmax(axis=-1) self.seq_length = config.seq_length self.hidden_size = config.hidden_size self.cast = ops.Cast() self.reshape = ops.Reshape() self.batch_matmul = ops.BatchMatMul() ones = np.ones(shape=(config.batch_size, config.seq_length, config.seq_length)) self.lower_triangle_mask = Tensor(np.tril(ones), dtype=mstype.float32) self.multiply = ops.Mul()
def __init__(self, embedding_size, embedding_shape, use_relative_positions=False, use_token_type=False, token_type_vocab_size=16, use_one_hot_embeddings=False, initializer_range=0.02, max_position_embeddings=512, dropout_prob=0.1): super(EmbeddingPostprocessor, self).__init__() self.use_token_type = use_token_type self.token_type_vocab_size = token_type_vocab_size self.use_one_hot_embeddings = use_one_hot_embeddings self.max_position_embeddings = max_position_embeddings self.embedding_table = Parameter(initializer (TruncatedNormal(initializer_range), [token_type_vocab_size, embedding_size]), name='embedding_table') self.shape_flat = (-1,) self.one_hot = ops.OneHot() self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.1, mstype.float32) self.array_mul = ops.MatMul() self.reshape = ops.Reshape() self.shape = tuple(embedding_shape) self.layernorm = nn.LayerNorm((embedding_size,)) self.dropout = nn.Dropout(1 - dropout_prob) self.gather = ops.GatherV2() self.use_relative_positions = use_relative_positions self.slice = ops.StridedSlice() self.full_position_embeddings = Parameter(initializer (TruncatedNormal(initializer_range), [max_position_embeddings, embedding_size]), name='full_position_embeddings')
def __init__(self, embedding_size, embedding_shape, use_relative_positions=False, use_token_type=False, token_type_vocab_size=16, use_one_hot_embeddings=False, initializer_range=0.02, max_position_embeddings=512, dropout_prob=0.1): super(EmbeddingPostprocessor, self).__init__() self.use_token_type = use_token_type self.token_type_vocab_size = token_type_vocab_size self.use_one_hot_embeddings = use_one_hot_embeddings self.max_position_embeddings = max_position_embeddings self.token_type_embedding = nn.Embedding( vocab_size=token_type_vocab_size, embedding_size=embedding_size, use_one_hot=use_one_hot_embeddings) self.shape_flat = (-1,) self.one_hot = P.OneHot() self.on_value = Tensor(1.0, mstype.float32) self.off_value = Tensor(0.1, mstype.float32) self.array_mul = P.MatMul() self.reshape = P.Reshape() self.shape = tuple(embedding_shape) self.dropout = nn.Dropout(1 - dropout_prob) self.gather = P.Gather() self.use_relative_positions = use_relative_positions self.slice = P.StridedSlice() _, seq, _ = self.shape self.full_position_embedding = nn.Embedding( vocab_size=max_position_embeddings, embedding_size=embedding_size, use_one_hot=False) self.layernorm = nn.LayerNorm((embedding_size,), epsilon=1e-5) self.position_ids = Tensor(np.arange(seq).reshape(-1, seq).astype(np.int32)) self.add = P.Add()
def __init__(self): super(Net, self).__init__() self.matmul = ops.MatMul() self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z')
from mindspore.common.api import ms_function import numpy as np from mindspore import Tensor from mindspore import context, Model from mindspore import ops context.set_context(mode=context.GRAPH_MODE, device_target='CPU') #context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU') x = Tensor(np.random.randn(4, 10), ms.float32) y = Tensor(np.random.randn(10, 10), ms.float32) minval = Tensor(0, mstype.int32) maxval = Tensor(100, mstype.int32) threshold = Tensor(50, mstype.int32) matmul = ops.MatMul() @ms_function def loop_fn(x, y): for i in range(10): x = matmul(x, y) return x print(loop_fn(x, y)) # failed to make the following codes work @ms_function def loop_and_cond_fn(x, y):