def get_bprop_matrix_inverse(self):
"""Grad definition for `MatrixInverse` operation."""
matmul_x1 = nn.MatMul(transpose_x1=True)
matmul_x2 = nn.MatMul(transpose_x2=True)
neg = P.Neg()
def bprop(x, out, dout):
dx = matmul_x2(dout, out)
dx = matmul_x1(out, dx)
dx = neg(dx)
return (dx,)
return bprop
def __init__(self, batch_size, query_linear_bias, key_linear_bias,
value_linear_bias):
"""init function"""
super(MultiHeadAttn, self).__init__()
self.batch_size = batch_size
self.matmul = nn.MatMul()
self.add = P.Add()
self.reshape = P.Reshape()
self.transpose = P.Transpose()
self.div = P.Div()
self.softmax = nn.Softmax(axis=3)
self.query_linear_weight = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)),
name=None)
self.query_linear_bias = query_linear_bias
self.key_linear_weight = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)),
name=None)
self.key_linear_bias = key_linear_bias
self.value_linear_weight = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)),
name=None)
self.value_linear_bias = value_linear_bias
self.reshape_shape = tuple([batch_size, 512, 64, 64])
self.w = Parameter(Tensor(
np.random.uniform(0, 1, (64, 64, 4096)).astype(np.float32)),
name=None)
self.b = Parameter(Tensor(
np.random.uniform(0, 1, (4096, )).astype(np.float32)),
name=None)
def construct(self, x):
out_shape = P.Shape()(x)[:-1] + (self.output_size, )
x = P.Reshape()(x, (-1, self.input_size))
x = nn.MatMul()(x, self.cast(self.weight, self.dtype)) + self.cast(
self.bias, self.dtype)
output = P.Reshape()(x, out_shape)
return output
def __init__(self, input_size, batch_size=64, hidden_size=512, num_class=11):
super(StackedRNN, self).__init__()
self.input_size = input_size
self.batch_size = batch_size
self.hidden_size = hidden_size
self.num_class = num_class
k = (1 / hidden_size) ** 0.5
self.rnn1 = P.DynamicRNN(forget_bias=0.0)
self.rnn2 = P.DynamicRNN(forget_bias=0.0)
self.w1 = Parameter(np.random.uniform(-k, k, (input_size + hidden_size, 4 * hidden_size)).astype(np.float16),
name="w1")
self.w2 = Parameter(np.random.uniform(-k, k, (hidden_size + hidden_size, 4 * hidden_size)).astype(np.float16),
name="w2")
self.b1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype(np.float16), name="b1")
self.b2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype(np.float16), name="b2")
self.h1 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16))
self.h2 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16))
self.c1 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16))
self.c2 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16))
self.fc_weight = Tensor(np.random.random((hidden_size, num_class)).astype(np.float16))
self.fc_bias = Tensor(np.random.random(self.num_class).astype(np.float16))
self.reshape = P.Reshape()
self.transpose = P.Transpose()
self.matmul = nn.MatMul()
def __init__(self, batch_size, passthrough_w_0, passthrough_w_1,
passthrough_w_2):
"""init function"""
super(MultiHeadAttn, self).__init__()
self.batch_size = batch_size
self.matmul_0 = nn.MatMul()
self.matmul_0_w = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)),
name=None)
self.matmul_1 = nn.MatMul()
self.matmul_1_w = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)),
name=None)
self.matmul_2 = nn.MatMul()
self.matmul_2_w = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)),
name=None)
self.add_3 = P.Add()
self.add_3_bias = passthrough_w_0
self.add_4 = P.Add()
self.add_4_bias = passthrough_w_1
self.add_5 = P.Add()
self.add_5_bias = passthrough_w_2
self.reshape_6 = P.Reshape()
self.reshape_6_shape = tuple([batch_size, 512, 64, 64])
self.reshape_7 = P.Reshape()
self.reshape_7_shape = tuple([batch_size, 512, 64, 64])
self.reshape_8 = P.Reshape()
self.reshape_8_shape = tuple([batch_size, 512, 64, 64])
self.transpose_9 = P.Transpose()
self.transpose_10 = P.Transpose()
self.transpose_11 = P.Transpose()
self.matmul_12 = nn.MatMul()
self.div_13 = P.Div()
self.div_13_w = 8.0
self.add_14 = P.Add()
self.softmax_15 = nn.Softmax(axis=3)
self.matmul_16 = nn.MatMul()
self.transpose_17 = P.Transpose()
self.matmul_18 = P.MatMul()
self.matmul_18_weight = Parameter(Tensor(
np.random.uniform(0, 1, (64, 64, 4096)).astype(np.float32)),
name=None)
self.add_19 = P.Add()
self.add_19_bias = Parameter(Tensor(
np.random.uniform(0, 1, (4096, )).astype(np.float32)),
name=None)
def __init__(self, matmul_0_weight_shape, add_1_bias_shape):
"""init function"""
super(Linear, self).__init__()
self.matmul_0 = nn.MatMul()
self.matmul_0_w = Parameter(Tensor(np.random.uniform(0, 1, matmul_0_weight_shape).astype(np.float32)),
name=None)
self.add_1 = P.Add()
self.add_1_bias = Parameter(Tensor(np.random.uniform(0, 1, add_1_bias_shape).astype(np.float32)), name=None)
def __init__(self, linear_weight_shape, linear_bias):
"""init function"""
super(Linear, self).__init__()
self.matmul = nn.MatMul()
self.add = P.Add()
self.weight = Parameter(Tensor(
np.random.uniform(0, 1, linear_weight_shape).astype(np.float32)),
name=None)
self.bias = linear_bias
def __init__(self, w_shape, b_shape):
super(Linear, self).__init__()
self.matmul = nn.MatMul()
self.matmul.to_float(mstype.float16)
self.w = Parameter(Tensor(
np.random.uniform(0, 1, w_shape).astype(np.float32)),
name=None)
self.add = P.Add()
self.b = Parameter(Tensor(
np.random.uniform(0, 1, b_shape).astype(np.float32)),
name=None)
def __init__(self, linear_1_weight_shape, linear_1_bias_shape,
bert_layer_norm_weight_shape, bert_layer_norm_bias_shape):
"""init function"""
super(SupportingOutputLayer, self).__init__()
self.linear_1 = Linear(linear_weight_shape=linear_1_weight_shape,
linear_bias_shape=linear_1_bias_shape)
self.bert_layer_norm = BertLayerNorm(
bert_layer_norm_weight_shape=bert_layer_norm_weight_shape,
bert_layer_norm_bias_shape=bert_layer_norm_bias_shape)
self.matmul = nn.MatMul()
self.matmul_w = Parameter(Tensor(
np.random.uniform(0, 1, (8192, 1)).astype(np.float32)),
name=None)
def __init__(self):
super(MultiHeadAttn, self).__init__()
self.matmul_0 = nn.MatMul()
self.matmul_0.to_float(mstype.float16)
self.matmul_0_w = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.matmul_1 = nn.MatMul()
self.matmul_1.to_float(mstype.float16)
self.matmul_1_w = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.matmul_2 = nn.MatMul()
self.matmul_2.to_float(mstype.float16)
self.matmul_2_w = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.add_3 = P.Add()
self.add_3_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
self.add_4 = P.Add()
self.add_4_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
self.add_5 = P.Add()
self.add_5_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
self.reshape_6 = P.Reshape()
self.reshape_6_shape = tuple([BATCH_SIZE, 448, 12, 64])
self.reshape_7 = P.Reshape()
self.reshape_7_shape = tuple([BATCH_SIZE, 448, 12, 64])
self.reshape_8 = P.Reshape()
self.reshape_8_shape = tuple([BATCH_SIZE, 448, 12, 64])
self.transpose_9 = P.Transpose()
self.transpose_10 = P.Transpose()
self.transpose_11 = P.Transpose()
self.matmul_12 = nn.MatMul()
self.matmul_12.to_float(mstype.float16)
self.div_13 = P.Div()
self.div_13_w = 8.0
self.add_14 = P.Add()
self.softmax_15 = nn.Softmax(axis=3)
self.matmul_16 = nn.MatMul()
self.matmul_16.to_float(mstype.float16)
self.transpose_17 = P.Transpose()
self.reshape_18 = P.Reshape()
self.reshape_18_shape = tuple([BATCH_SIZE, 448, 768])
self.matmul_19 = nn.MatMul()
self.matmul_19.to_float(mstype.float16)
self.matmul_19_w = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.add_20 = P.Add()
self.add_20_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
def __init__(self, module15_0_matmul_0_weight_shape,
module15_0_add_1_bias_shape, normmodule_0_mul_8_w_shape,
normmodule_0_add_9_bias_shape):
"""init function"""
super(Module16, self).__init__()
self.module15_0 = Module15(
matmul_0_weight_shape=module15_0_matmul_0_weight_shape,
add_1_bias_shape=module15_0_add_1_bias_shape)
self.normmodule_0 = NormModule(
mul_8_w_shape=normmodule_0_mul_8_w_shape,
add_9_bias_shape=normmodule_0_add_9_bias_shape)
self.matmul_0 = nn.MatMul()
self.matmul_0_w = Parameter(Tensor(
np.random.uniform(0, 1, (8192, 1)).astype(np.float32)),
name=None)
def __init__(self):
super(Processing, self).__init__()
self.slice = P.Slice()
self.reshape = P.Reshape()
self.shape = P.Shape()
self.batchmat = nn.MatMul()
self.split = P.Split(1, 3)
self.concat = P.Concat(axis=1)
slice_64 = Tensor(np.hstack((np.identity(64), np.zeros([64, 128]))))
slice_128 = Tensor(np.hstack((np.zeros([128, 64]), np.identity(128))))
self.slice_0 = [slice_64, slice_128]
slice_46 = Tensor(np.hstack((np.identity(46), np.zeros([46, 92]))))
slice_92 = Tensor(np.hstack((np.zeros([92, 46]), np.identity(92))))
self.slice_1 = [slice_46, slice_92]
slice_2 = np.vstack((np.identity(1), np.zeros([3, 1])))
self.slice_2 = Tensor(slice_2)
def __init__(self, linear_weight_shape, linear_bias_shape,
bert_layer_norm_weight_shape, bert_layer_norm_bias_shape):
"""init function"""
super(PosOutputLayer, self).__init__()
self.linear_1 = Linear(linear_weight_shape=linear_weight_shape,
linear_bias_shape=linear_bias_shape)
self.bert_layer_norm = BertLayerNorm(
bert_layer_norm_weight_shape=bert_layer_norm_weight_shape,
bert_layer_norm_bias_shape=bert_layer_norm_bias_shape)
self.matmul = nn.MatMul()
self.linear_2_weight = Parameter(Tensor(
np.random.uniform(0, 1, (4096, 1)).astype(np.float32)),
name=None)
self.add = P.Add()
self.linear_2_bias = Parameter(Tensor(
np.random.uniform(0, 1, (1, )).astype(np.float32)),
name=None)
def __init__(
self,
hidden_size,
vocab_size,
sample_softmax,
num_sampled,
num_true=1,
seed=0,
training=True):
super().__init__()
self.training = training
self.sample_softmax = sample_softmax
self.hidden_size = hidden_size
self.weight = Parameter(initializer(Normal(1.0 / np.sqrt(hidden_size)), (vocab_size, hidden_size), mindspore.float32))
self.bias = Parameter(initializer(Zero(), (vocab_size), mindspore.float32))
self.sampled_softmax_loss = SampledSoftmaxLoss(num_sampled, vocab_size, num_true, seed=seed)
self.sparse_softmax_cross_entropy_with_logits = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
self.matmul = nn.MatMul(False, True)
self.reduce_mean = P.ReduceMean()
def __init__(self, batch_size, temperature=1, world_size=1):
super(NT_Xent_Loss, self).__init__()
# Parameters.
self.LARGE_NUM = 1e9
self.batch_size = batch_size
self.temperature = temperature
self.world_size = world_size
self.N = 2 * self.batch_size * self.world_size
# Tail_Loss.
self.criterion = CrossEntropyLoss(reduction="mean")
self.norm = P.L2Normalize(axis=1)
self.one_hot = P.OneHot()
self.range = nn.Range(0, self.batch_size)
self.one = Tensor(1.0, mstype.float32)
self.zero = Tensor(0.0, mstype.float32)
self.transpose = P.Transpose()
self.matmul = nn.MatMul()
# Operations.
self.ones = P.Ones()
self.zeros = P.Zeros()
self.cat1 = P.Concat(axis=1)
def __init__(self, seq_len):
super(MultiHeadAttn, self).__init__()
self.matmul = nn.MatMul()
self.matmul.to_float(mstype.float16)
self.query = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.key = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.value = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.add = P.Add()
self.query_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
self.key_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
self.value_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
self.reshape = P.Reshape()
self.to_shape_0 = tuple([BATCH_SIZE, seq_len, 12, 64])
self.transpose = P.Transpose()
self.div = P.Div()
self.div_w = 8.0
self.softmax = nn.Softmax(axis=3)
self.to_shape_1 = tuple([BATCH_SIZE, seq_len, 768])
self.context_weight = Parameter(Tensor(
np.random.uniform(0, 1, (768, 768)).astype(np.float32)),
name=None)
self.context_bias = Parameter(Tensor(
np.random.uniform(0, 1, (768, )).astype(np.float32)),
name=None)
def __init__(self):
super(MDNet, self).__init__()
self.reshape = P.Reshape()
self.shape = P.Shape()
self.concat0 = P.Concat(axis=0)
self.tanh = nn.Tanh()
self.mat = P.MatMul()
self.batchmat = nn.MatMul()
self.batchmat_tran = nn.MatMul(transpose_x1=True)
self.idt1 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )),
dtype=mstype.float32),
name="type0_idt1")
self.idt2 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )),
dtype=mstype.float32),
name="type0_idt2")
self.idt3 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )),
dtype=mstype.float32),
name="type1_idt1")
self.idt4 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )),
dtype=mstype.float32),
name="type1_idt2")
self.idt = [self.idt1, self.idt2, self.idt3, self.idt4]
self.neuron = [dim_descrpt] + n_neuron
self.par = [1] + filter_neuron
self.process = Processing()
fc = []
for i in range(3):
fc.append(
nn.Dense(self.par[i],
self.par[i + 1],
weight_init=Tensor(np.random.normal(
0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]),
(self.par[i + 1], self.par[i])),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(
0.0, 1.0, (self.par[i + 1], )),
dtype=mstype.float32)))
for i in range(1, 3):
fc.append(
nn.Dense(self.par[i],
self.par[i + 1],
weight_init=Tensor(np.random.normal(
0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]),
(self.par[i + 1], self.par[i])),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(
0.0, 1.0, (self.par[i + 1], )),
dtype=mstype.float32)))
for i in range(3):
fc.append(
nn.Dense(self.par[i],
self.par[i + 1],
weight_init=Tensor(np.random.normal(
0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]),
(self.par[i + 1], self.par[i])),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(
0.0, 1.0, (self.par[i + 1], )),
dtype=mstype.float32)))
for i in range(1, 3):
fc.append(
nn.Dense(self.par[i],
self.par[i + 1],
weight_init=Tensor(np.random.normal(
0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]),
(self.par[i + 1], self.par[i])),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(
0.0, 1.0, (self.par[i + 1], )),
dtype=mstype.float32)))
self.fc = nn.CellList(fc)
self.fc0 = deepcopy(self.fc)
self.fc2 = [self.fc, self.fc0]
fc = []
for i in range(3):
fc.append(
nn.Dense(
self.neuron[i],
self.neuron[i + 1],
weight_init=Tensor(np.random.normal(
0.0,
1.0 / np.sqrt(self.neuron[i] + self.neuron[i + 1]),
(self.neuron[i + 1], self.neuron[i])),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(0.0, 1.0,
(self.neuron[i + 1], )),
dtype=mstype.float32)))
fc.append(
nn.Dense(240,
1,
weight_init=Tensor(np.random.normal(
0.0, 1.0 / np.sqrt(240 + 1), (1, 240)),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(type_bias_ae[0], 1.0,
(1, )),
dtype=mstype.float32)))
for i in range(3):
fc.append(
nn.Dense(
self.neuron[i],
self.neuron[i + 1],
weight_init=Tensor(np.random.normal(
0.0,
1.0 / np.sqrt(self.neuron[i] + self.neuron[i + 1]),
(self.neuron[i + 1], self.neuron[i])),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(0.0, 1.0,
(self.neuron[i + 1], )),
dtype=mstype.float32)))
fc.append(
nn.Dense(240,
1,
weight_init=Tensor(np.random.normal(
0.0, 1.0 / np.sqrt(240 + 1), (1, 240)),
dtype=mstype.float32),
bias_init=Tensor(np.random.normal(type_bias_ae[1], 1.0,
(1, )),
dtype=mstype.float32)))
self.fc1 = nn.CellList(fc)
xyz_A = np.vstack((np.identity(46), np.zeros([92, 46])))
self.xyz_A = Tensor(np.reshape(xyz_A, (1, 138, 46)))
xyz_B = np.vstack((np.zeros([46, 92]), np.identity(92)))
self.xyz_B = Tensor(np.reshape(xyz_B, (1, 138, 92)))
xyz_2 = np.vstack(
(np.identity(n_axis_neuron),
np.zeros([self.par[-1] - n_axis_neuron, n_axis_neuron])))
self.xyz_2 = Tensor(xyz_2)
def __init__(self, transpose_x1, transpose_x2):
super(Net, self).__init__()
self.matmul = nn.MatMul(transpose_x1, transpose_x2)
