def rnn_relu_cell(input, hidden, w_ih, w_hh, b_ih, b_hh):
if b_ih is None:
igates = P.MatMul(False, True)(input, w_ih)
hgates = P.MatMul(False, True)(hidden, w_hh)
else:
igates = P.MatMul(False, True)(input, w_ih) + b_ih
hgates = P.MatMul(False, True)(hidden, w_hh) + b_hh
return P.ReLU()(igates + hgates)
def rnn_relu_cell(inputs, hidden, w_ih, w_hh, b_ih, b_hh):
'''RNN cell function with relu activation'''
if b_ih is None:
igates = P.MatMul(False, True)(inputs, w_ih)
hgates = P.MatMul(False, True)(hidden, w_hh)
else:
igates = P.MatMul(False, True)(inputs, w_ih) + b_ih
hgates = P.MatMul(False, True)(hidden, w_hh) + b_hh
return P.ReLU()(igates + hgates)
def __init__(self, config):
super(GetMaskedLMOutput, self).__init__()
self.width = config.hidden_size
self.reshape = ops.Reshape()
self.gather = ops.GatherV2()
weight_init = TruncatedNormal(config.initializer_range)
self.dense = nn.Dense(self.width,
config.hidden_size,
weight_init=weight_init,
activation=config.hidden_act).to_float(config.compute_type)
self.layernorm = nn.LayerNorm((config.hidden_size,)).to_float(config.compute_type)
self.output_bias = Parameter(
initializer(
'zero',
config.vocab_size),
name='output_bias')
self.matmul = ops.MatMul(transpose_b=True)
self.log_softmax = nn.LogSoftmax(axis=-1)
self.shape_flat_offsets = (-1, 1)
self.rng = Tensor(np.array(range(0, config.batch_size)).astype(np.int32))
self.last_idx = (-1,)
self.shape_flat_sequence_tensor = (config.batch_size * config.seq_length, self.width)
self.seq_length_tensor = Tensor(np.array((config.seq_length,)).astype(np.int32))
self.cast = ops.Cast()
self.compute_type = config.compute_type
self.dtype = config.dtype
def __init__(self,
input_size: int,
hidden_size: int,
bias: bool = True,
nonlinearity: str = 'tanh'):
super().__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.bias = bias
stdv = 1 / math.sqrt(hidden_size)
self.weight_ih = Parameter(
Tensor(
np.random.uniform(-stdv, stdv,
(input_size, hidden_size)).astype(
np.float32)))
self.weight_hh = Parameter(
Tensor(
np.random.uniform(-stdv, stdv,
(hidden_size, hidden_size)).astype(
np.float32)))
if bias:
self.bias_ih = Parameter(
Tensor(
np.random.uniform(-stdv, stdv,
(hidden_size)).astype(np.float32)))
self.bias_hh = Parameter(
Tensor(
np.random.uniform(-stdv, stdv,
(hidden_size)).astype(np.float32)))
self.nonlinearity = self.nonlinearity_dict[nonlinearity]
self.mm = P.MatMul()
def gru_cell(input, hidden, w_ih, w_hh, b_ih, b_hh):
if b_ih is None:
gi = P.MatMul(False, True)(input, w_ih)
gh = P.MatMul(False, True)(hidden, w_hh)
else:
gi = P.MatMul(False, True)(input, w_ih) + b_ih
gh = P.MatMul(False, True)(hidden, w_hh) + b_hh
i_r, i_i, i_n = P.Split(1, 3)(gi)
h_r, h_i, h_n = P.Split(1, 3)(gh)
resetgate = P.Sigmoid()(i_r + h_r)
inputgate = P.Sigmoid()(i_i + h_i)
newgate = P.Tanh()(i_n + resetgate * h_n)
hy = newgate + inputgate * (hidden - newgate)
return hy
def lstm_cell(input, hidden, w_ih, w_hh, b_ih, b_hh):
hx, cx = hidden
if b_ih is None:
gates = P.MatMul(False, True)(input, w_ih) + P.MatMul(False, True)(hx, w_hh)
else:
gates = P.MatMul(False, True)(input, w_ih) + P.MatMul(False, True)(hx, w_hh) + b_ih + b_hh
ingate, forgetgate, cellgate, outgate = P.Split(1, 4)(gates)
ingate = P.Sigmoid()(ingate)
forgetgate = P.Sigmoid()(forgetgate)
cellgate = P.Tanh()(cellgate)
outgate = P.Sigmoid()(outgate)
cy = (forgetgate * cx) + (ingate * cellgate)
hy = outgate * P.Tanh()(cy)
return hy, cy
def __init__(self, matmul_size, transpose_a=False, transpose_b=False, strategy=None):
"""init"""
super().__init__()
matmul_np = np.full(matmul_size, 0.5, dtype=np.float32)
self.matmul_weight = Parameter(Tensor(matmul_np))
self.matmul = ops.MatMul(transpose_a=transpose_a, transpose_b=transpose_b)
self.neg = ops.Neg()
if strategy is not None:
self.matmul.shard(strategy)
def __init__(self,
in1_channels,
in2_channels,
out_channels,
weight_init=None,
bias_init=None,
has_bias=True):
super().__init__()
# self.in_channels = Validator.check_positive_int(in1_channels, "in1_channels", self.cls_name)
# self.in_channels = Validator.check_positive_int(in2_channels, "in2_channels", self.cls_name)
# self.out_channels = Validator.check_positive_int(out_channels, "out_channels", self.cls_name)
# self.has_bias = Validator.check_bool(has_bias, "has_bias", self.cls_name)
self.in1_channels = in1_channels
self.in2_channels = in2_channels
self.out_channels = out_channels
self.has_bias = has_bias
bound = 1 / math.sqrt(in1_channels)
if weight_init is None:
weight_init = Uniform(bound)
if isinstance(weight_init, Tensor):
if weight_init.ndim != 3 or weight_init.shape[0] != out_channels or \
weight_init.shape[1] != in1_channels or weight_init.shape[2] != in2_channels:
raise ValueError(f"For '{self.cls_name}', weight init shape error. The ndim of 'weight_init' must "
f"be equal to 3, the first dim must be equal to 'out_channels', the "
f"second dim must be equal to 'in1_channels', and the third dim must be "
f"equal to 'in2_channels'. But got 'weight_init': {weight_init}, "
f"'out_channels': {out_channels}, 'in_channels': {in1_channels}, "
f"'in2_channels': {in2_channels}")
self.weight = Parameter(initializer(weight_init, (out_channels, in1_channels, in2_channels)), 'weight')
if self.has_bias:
if bias_init is None:
bias_init = Uniform(bound)
if isinstance(bias_init, Tensor):
if bias_init.ndim != 1 or bias_init.shape[0] != out_channels:
raise ValueError(f"For '{self.cls_name}', bias init shape error. The ndim of 'bias_init' should "
f"be equal to 1, and the first dim must be equal to 'out_channels'. But got "
f"'bias_init': {bias_init}, 'out_channels': {out_channels}.")
self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias")
self.bias_add = P.BiasAdd()
self.matmul = P.MatMul()
def __init__(self, input_size: int, hidden_size: int, bias: bool = True, \
cell_clip=None, proj_size=None, proj_clip=None):
super().__init__(input_size,
hidden_size,
bias,
num_chunks=4,
proj_size=proj_size)
self.cell_clip = cell_clip
self.proj_size = proj_size
self.proj_clip = proj_clip
if proj_size is not None:
self.proj_weight = Parameter(
Tensor(
np.random.randn(hidden_size,
proj_size).astype(np.float32)))
self.matmul = P.MatMul()
self.cell_type = 'LSTM'
def __init__(self,
vocab_size,
embedding_size,
embedding_shape,
use_one_hot_embeddings=False,
initializer_range=0.02):
super(EmbeddingLookup, self).__init__()
self.vocab_size = vocab_size
self.use_one_hot_embeddings = use_one_hot_embeddings
self.embedding_table = Parameter(initializer
(TruncatedNormal(initializer_range),
[vocab_size, embedding_size]))
self.expand = P.ExpandDims()
self.shape_flat = (-1,)
self.gather = P.Gather()
self.one_hot = P.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.array_mul = P.MatMul()
self.reshape = P.Reshape()
self.shape = tuple(embedding_shape)
def __init__(self, config, is_training, num_tokens, dropout_prob=0.0, use_one_hot_embeddings=False):
super(BertPoetryModel, self).__init__()
self.bert = BertModel(config, is_training, use_one_hot_embeddings)
self.num_tokens = num_tokens
idx = np.arange(config.seq_length)
mask = idx[None, :] <= idx[:, None]
self.mask = Tensor([mask], mstype.float32)
self.MLM_Dense = nn.Dense(config.hidden_size, config.hidden_size,\
has_bias=True, weight_init=TruncatedNormal(0.02),\
activation='gelu').to_float(mstype.float16)
self.layer_norm = nn.LayerNorm((config.hidden_size,))
self.matmul = ops.MatMul(transpose_b=True)
self.biasadd = Parameter(initializer('zero', self.num_tokens), name='MLM_output_biasadd')
self.softmax = ops.Softmax(axis=-1)
self.seq_length = config.seq_length
self.hidden_size = config.hidden_size
self.cast = ops.Cast()
self.reshape = ops.Reshape()
self.batch_matmul = ops.BatchMatMul()
ones = np.ones(shape=(config.batch_size, config.seq_length, config.seq_length))
self.lower_triangle_mask = Tensor(np.tril(ones), dtype=mstype.float32)
self.multiply = ops.Mul()
def __init__(self,
embedding_size,
embedding_shape,
use_relative_positions=False,
use_token_type=False,
token_type_vocab_size=16,
use_one_hot_embeddings=False,
initializer_range=0.02,
max_position_embeddings=512,
dropout_prob=0.1):
super(EmbeddingPostprocessor, self).__init__()
self.use_token_type = use_token_type
self.token_type_vocab_size = token_type_vocab_size
self.use_one_hot_embeddings = use_one_hot_embeddings
self.max_position_embeddings = max_position_embeddings
self.embedding_table = Parameter(initializer
(TruncatedNormal(initializer_range),
[token_type_vocab_size,
embedding_size]),
name='embedding_table')
self.shape_flat = (-1,)
self.one_hot = ops.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.1, mstype.float32)
self.array_mul = ops.MatMul()
self.reshape = ops.Reshape()
self.shape = tuple(embedding_shape)
self.layernorm = nn.LayerNorm((embedding_size,))
self.dropout = nn.Dropout(1 - dropout_prob)
self.gather = ops.GatherV2()
self.use_relative_positions = use_relative_positions
self.slice = ops.StridedSlice()
self.full_position_embeddings = Parameter(initializer
(TruncatedNormal(initializer_range),
[max_position_embeddings,
embedding_size]),
name='full_position_embeddings')
def __init__(self,
embedding_size,
embedding_shape,
use_relative_positions=False,
use_token_type=False,
token_type_vocab_size=16,
use_one_hot_embeddings=False,
initializer_range=0.02,
max_position_embeddings=512,
dropout_prob=0.1):
super(EmbeddingPostprocessor, self).__init__()
self.use_token_type = use_token_type
self.token_type_vocab_size = token_type_vocab_size
self.use_one_hot_embeddings = use_one_hot_embeddings
self.max_position_embeddings = max_position_embeddings
self.token_type_embedding = nn.Embedding(
vocab_size=token_type_vocab_size,
embedding_size=embedding_size,
use_one_hot=use_one_hot_embeddings)
self.shape_flat = (-1,)
self.one_hot = P.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.1, mstype.float32)
self.array_mul = P.MatMul()
self.reshape = P.Reshape()
self.shape = tuple(embedding_shape)
self.dropout = nn.Dropout(1 - dropout_prob)
self.gather = P.Gather()
self.use_relative_positions = use_relative_positions
self.slice = P.StridedSlice()
_, seq, _ = self.shape
self.full_position_embedding = nn.Embedding(
vocab_size=max_position_embeddings,
embedding_size=embedding_size,
use_one_hot=False)
self.layernorm = nn.LayerNorm((embedding_size,), epsilon=1e-5)
self.position_ids = Tensor(np.arange(seq).reshape(-1, seq).astype(np.int32))
self.add = P.Add()
def __init__(self):
super(Net, self).__init__()
self.matmul = ops.MatMul()
self.z = Parameter(Tensor(np.array([1.0], np.float32)), name='z')
from mindspore.common.api import ms_function
import numpy as np
from mindspore import Tensor
from mindspore import context, Model
from mindspore import ops
context.set_context(mode=context.GRAPH_MODE, device_target='CPU')
#context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU')
x = Tensor(np.random.randn(4, 10), ms.float32)
y = Tensor(np.random.randn(10, 10), ms.float32)
minval = Tensor(0, mstype.int32)
maxval = Tensor(100, mstype.int32)
threshold = Tensor(50, mstype.int32)
matmul = ops.MatMul()
@ms_function
def loop_fn(x, y):
for i in range(10):
x = matmul(x, y)
return x
print(loop_fn(x, y))
# failed to make the following codes work
@ms_function
def loop_and_cond_fn(x, y):
