def get_bprop_matrix_inverse(self): """Grad definition for `MatrixInverse` operation.""" matmul_x1 = nn.MatMul(transpose_x1=True) matmul_x2 = nn.MatMul(transpose_x2=True) neg = P.Neg() def bprop(x, out, dout): dx = matmul_x2(dout, out) dx = matmul_x1(out, dx) dx = neg(dx) return (dx,) return bprop
def __init__(self, batch_size, query_linear_bias, key_linear_bias, value_linear_bias): """init function""" super(MultiHeadAttn, self).__init__() self.batch_size = batch_size self.matmul = nn.MatMul() self.add = P.Add() self.reshape = P.Reshape() self.transpose = P.Transpose() self.div = P.Div() self.softmax = nn.Softmax(axis=3) self.query_linear_weight = Parameter(Tensor( np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)), name=None) self.query_linear_bias = query_linear_bias self.key_linear_weight = Parameter(Tensor( np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)), name=None) self.key_linear_bias = key_linear_bias self.value_linear_weight = Parameter(Tensor( np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)), name=None) self.value_linear_bias = value_linear_bias self.reshape_shape = tuple([batch_size, 512, 64, 64]) self.w = Parameter(Tensor( np.random.uniform(0, 1, (64, 64, 4096)).astype(np.float32)), name=None) self.b = Parameter(Tensor( np.random.uniform(0, 1, (4096, )).astype(np.float32)), name=None)
def construct(self, x): out_shape = P.Shape()(x)[:-1] + (self.output_size, ) x = P.Reshape()(x, (-1, self.input_size)) x = nn.MatMul()(x, self.cast(self.weight, self.dtype)) + self.cast( self.bias, self.dtype) output = P.Reshape()(x, out_shape) return output
def __init__(self, input_size, batch_size=64, hidden_size=512, num_class=11): super(StackedRNN, self).__init__() self.input_size = input_size self.batch_size = batch_size self.hidden_size = hidden_size self.num_class = num_class k = (1 / hidden_size) ** 0.5 self.rnn1 = P.DynamicRNN(forget_bias=0.0) self.rnn2 = P.DynamicRNN(forget_bias=0.0) self.w1 = Parameter(np.random.uniform(-k, k, (input_size + hidden_size, 4 * hidden_size)).astype(np.float16), name="w1") self.w2 = Parameter(np.random.uniform(-k, k, (hidden_size + hidden_size, 4 * hidden_size)).astype(np.float16), name="w2") self.b1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype(np.float16), name="b1") self.b2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype(np.float16), name="b2") self.h1 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.h2 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.c1 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.c2 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.fc_weight = Tensor(np.random.random((hidden_size, num_class)).astype(np.float16)) self.fc_bias = Tensor(np.random.random(self.num_class).astype(np.float16)) self.reshape = P.Reshape() self.transpose = P.Transpose() self.matmul = nn.MatMul()
def __init__(self, batch_size, passthrough_w_0, passthrough_w_1, passthrough_w_2): """init function""" super(MultiHeadAttn, self).__init__() self.batch_size = batch_size self.matmul_0 = nn.MatMul() self.matmul_0_w = Parameter(Tensor( np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)), name=None) self.matmul_1 = nn.MatMul() self.matmul_1_w = Parameter(Tensor( np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)), name=None) self.matmul_2 = nn.MatMul() self.matmul_2_w = Parameter(Tensor( np.random.uniform(0, 1, (4096, 4096)).astype(np.float32)), name=None) self.add_3 = P.Add() self.add_3_bias = passthrough_w_0 self.add_4 = P.Add() self.add_4_bias = passthrough_w_1 self.add_5 = P.Add() self.add_5_bias = passthrough_w_2 self.reshape_6 = P.Reshape() self.reshape_6_shape = tuple([batch_size, 512, 64, 64]) self.reshape_7 = P.Reshape() self.reshape_7_shape = tuple([batch_size, 512, 64, 64]) self.reshape_8 = P.Reshape() self.reshape_8_shape = tuple([batch_size, 512, 64, 64]) self.transpose_9 = P.Transpose() self.transpose_10 = P.Transpose() self.transpose_11 = P.Transpose() self.matmul_12 = nn.MatMul() self.div_13 = P.Div() self.div_13_w = 8.0 self.add_14 = P.Add() self.softmax_15 = nn.Softmax(axis=3) self.matmul_16 = nn.MatMul() self.transpose_17 = P.Transpose() self.matmul_18 = P.MatMul() self.matmul_18_weight = Parameter(Tensor( np.random.uniform(0, 1, (64, 64, 4096)).astype(np.float32)), name=None) self.add_19 = P.Add() self.add_19_bias = Parameter(Tensor( np.random.uniform(0, 1, (4096, )).astype(np.float32)), name=None)
def __init__(self, matmul_0_weight_shape, add_1_bias_shape): """init function""" super(Linear, self).__init__() self.matmul_0 = nn.MatMul() self.matmul_0_w = Parameter(Tensor(np.random.uniform(0, 1, matmul_0_weight_shape).astype(np.float32)), name=None) self.add_1 = P.Add() self.add_1_bias = Parameter(Tensor(np.random.uniform(0, 1, add_1_bias_shape).astype(np.float32)), name=None)
def __init__(self, linear_weight_shape, linear_bias): """init function""" super(Linear, self).__init__() self.matmul = nn.MatMul() self.add = P.Add() self.weight = Parameter(Tensor( np.random.uniform(0, 1, linear_weight_shape).astype(np.float32)), name=None) self.bias = linear_bias
def __init__(self, w_shape, b_shape): super(Linear, self).__init__() self.matmul = nn.MatMul() self.matmul.to_float(mstype.float16) self.w = Parameter(Tensor( np.random.uniform(0, 1, w_shape).astype(np.float32)), name=None) self.add = P.Add() self.b = Parameter(Tensor( np.random.uniform(0, 1, b_shape).astype(np.float32)), name=None)
def __init__(self, linear_1_weight_shape, linear_1_bias_shape, bert_layer_norm_weight_shape, bert_layer_norm_bias_shape): """init function""" super(SupportingOutputLayer, self).__init__() self.linear_1 = Linear(linear_weight_shape=linear_1_weight_shape, linear_bias_shape=linear_1_bias_shape) self.bert_layer_norm = BertLayerNorm( bert_layer_norm_weight_shape=bert_layer_norm_weight_shape, bert_layer_norm_bias_shape=bert_layer_norm_bias_shape) self.matmul = nn.MatMul() self.matmul_w = Parameter(Tensor( np.random.uniform(0, 1, (8192, 1)).astype(np.float32)), name=None)
def __init__(self): super(MultiHeadAttn, self).__init__() self.matmul_0 = nn.MatMul() self.matmul_0.to_float(mstype.float16) self.matmul_0_w = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.matmul_1 = nn.MatMul() self.matmul_1.to_float(mstype.float16) self.matmul_1_w = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.matmul_2 = nn.MatMul() self.matmul_2.to_float(mstype.float16) self.matmul_2_w = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.add_3 = P.Add() self.add_3_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None) self.add_4 = P.Add() self.add_4_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None) self.add_5 = P.Add() self.add_5_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None) self.reshape_6 = P.Reshape() self.reshape_6_shape = tuple([BATCH_SIZE, 448, 12, 64]) self.reshape_7 = P.Reshape() self.reshape_7_shape = tuple([BATCH_SIZE, 448, 12, 64]) self.reshape_8 = P.Reshape() self.reshape_8_shape = tuple([BATCH_SIZE, 448, 12, 64]) self.transpose_9 = P.Transpose() self.transpose_10 = P.Transpose() self.transpose_11 = P.Transpose() self.matmul_12 = nn.MatMul() self.matmul_12.to_float(mstype.float16) self.div_13 = P.Div() self.div_13_w = 8.0 self.add_14 = P.Add() self.softmax_15 = nn.Softmax(axis=3) self.matmul_16 = nn.MatMul() self.matmul_16.to_float(mstype.float16) self.transpose_17 = P.Transpose() self.reshape_18 = P.Reshape() self.reshape_18_shape = tuple([BATCH_SIZE, 448, 768]) self.matmul_19 = nn.MatMul() self.matmul_19.to_float(mstype.float16) self.matmul_19_w = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.add_20 = P.Add() self.add_20_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None)
def __init__(self, module15_0_matmul_0_weight_shape, module15_0_add_1_bias_shape, normmodule_0_mul_8_w_shape, normmodule_0_add_9_bias_shape): """init function""" super(Module16, self).__init__() self.module15_0 = Module15( matmul_0_weight_shape=module15_0_matmul_0_weight_shape, add_1_bias_shape=module15_0_add_1_bias_shape) self.normmodule_0 = NormModule( mul_8_w_shape=normmodule_0_mul_8_w_shape, add_9_bias_shape=normmodule_0_add_9_bias_shape) self.matmul_0 = nn.MatMul() self.matmul_0_w = Parameter(Tensor( np.random.uniform(0, 1, (8192, 1)).astype(np.float32)), name=None)
def __init__(self): super(Processing, self).__init__() self.slice = P.Slice() self.reshape = P.Reshape() self.shape = P.Shape() self.batchmat = nn.MatMul() self.split = P.Split(1, 3) self.concat = P.Concat(axis=1) slice_64 = Tensor(np.hstack((np.identity(64), np.zeros([64, 128])))) slice_128 = Tensor(np.hstack((np.zeros([128, 64]), np.identity(128)))) self.slice_0 = [slice_64, slice_128] slice_46 = Tensor(np.hstack((np.identity(46), np.zeros([46, 92])))) slice_92 = Tensor(np.hstack((np.zeros([92, 46]), np.identity(92)))) self.slice_1 = [slice_46, slice_92] slice_2 = np.vstack((np.identity(1), np.zeros([3, 1]))) self.slice_2 = Tensor(slice_2)
def __init__(self, linear_weight_shape, linear_bias_shape, bert_layer_norm_weight_shape, bert_layer_norm_bias_shape): """init function""" super(PosOutputLayer, self).__init__() self.linear_1 = Linear(linear_weight_shape=linear_weight_shape, linear_bias_shape=linear_bias_shape) self.bert_layer_norm = BertLayerNorm( bert_layer_norm_weight_shape=bert_layer_norm_weight_shape, bert_layer_norm_bias_shape=bert_layer_norm_bias_shape) self.matmul = nn.MatMul() self.linear_2_weight = Parameter(Tensor( np.random.uniform(0, 1, (4096, 1)).astype(np.float32)), name=None) self.add = P.Add() self.linear_2_bias = Parameter(Tensor( np.random.uniform(0, 1, (1, )).astype(np.float32)), name=None)
def __init__( self, hidden_size, vocab_size, sample_softmax, num_sampled, num_true=1, seed=0, training=True): super().__init__() self.training = training self.sample_softmax = sample_softmax self.hidden_size = hidden_size self.weight = Parameter(initializer(Normal(1.0 / np.sqrt(hidden_size)), (vocab_size, hidden_size), mindspore.float32)) self.bias = Parameter(initializer(Zero(), (vocab_size), mindspore.float32)) self.sampled_softmax_loss = SampledSoftmaxLoss(num_sampled, vocab_size, num_true, seed=seed) self.sparse_softmax_cross_entropy_with_logits = nn.SoftmaxCrossEntropyWithLogits(sparse=True) self.matmul = nn.MatMul(False, True) self.reduce_mean = P.ReduceMean()
def __init__(self, batch_size, temperature=1, world_size=1): super(NT_Xent_Loss, self).__init__() # Parameters. self.LARGE_NUM = 1e9 self.batch_size = batch_size self.temperature = temperature self.world_size = world_size self.N = 2 * self.batch_size * self.world_size # Tail_Loss. self.criterion = CrossEntropyLoss(reduction="mean") self.norm = P.L2Normalize(axis=1) self.one_hot = P.OneHot() self.range = nn.Range(0, self.batch_size) self.one = Tensor(1.0, mstype.float32) self.zero = Tensor(0.0, mstype.float32) self.transpose = P.Transpose() self.matmul = nn.MatMul() # Operations. self.ones = P.Ones() self.zeros = P.Zeros() self.cat1 = P.Concat(axis=1)
def __init__(self, seq_len): super(MultiHeadAttn, self).__init__() self.matmul = nn.MatMul() self.matmul.to_float(mstype.float16) self.query = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.key = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.value = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.add = P.Add() self.query_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None) self.key_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None) self.value_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None) self.reshape = P.Reshape() self.to_shape_0 = tuple([BATCH_SIZE, seq_len, 12, 64]) self.transpose = P.Transpose() self.div = P.Div() self.div_w = 8.0 self.softmax = nn.Softmax(axis=3) self.to_shape_1 = tuple([BATCH_SIZE, seq_len, 768]) self.context_weight = Parameter(Tensor( np.random.uniform(0, 1, (768, 768)).astype(np.float32)), name=None) self.context_bias = Parameter(Tensor( np.random.uniform(0, 1, (768, )).astype(np.float32)), name=None)
def __init__(self): super(MDNet, self).__init__() self.reshape = P.Reshape() self.shape = P.Shape() self.concat0 = P.Concat(axis=0) self.tanh = nn.Tanh() self.mat = P.MatMul() self.batchmat = nn.MatMul() self.batchmat_tran = nn.MatMul(transpose_x1=True) self.idt1 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )), dtype=mstype.float32), name="type0_idt1") self.idt2 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )), dtype=mstype.float32), name="type0_idt2") self.idt3 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )), dtype=mstype.float32), name="type1_idt1") self.idt4 = Parameter(Tensor(np.random.normal(0.1, 0.001, (240, )), dtype=mstype.float32), name="type1_idt2") self.idt = [self.idt1, self.idt2, self.idt3, self.idt4] self.neuron = [dim_descrpt] + n_neuron self.par = [1] + filter_neuron self.process = Processing() fc = [] for i in range(3): fc.append( nn.Dense(self.par[i], self.par[i + 1], weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]), (self.par[i + 1], self.par[i])), dtype=mstype.float32), bias_init=Tensor(np.random.normal( 0.0, 1.0, (self.par[i + 1], )), dtype=mstype.float32))) for i in range(1, 3): fc.append( nn.Dense(self.par[i], self.par[i + 1], weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]), (self.par[i + 1], self.par[i])), dtype=mstype.float32), bias_init=Tensor(np.random.normal( 0.0, 1.0, (self.par[i + 1], )), dtype=mstype.float32))) for i in range(3): fc.append( nn.Dense(self.par[i], self.par[i + 1], weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]), (self.par[i + 1], self.par[i])), dtype=mstype.float32), bias_init=Tensor(np.random.normal( 0.0, 1.0, (self.par[i + 1], )), dtype=mstype.float32))) for i in range(1, 3): fc.append( nn.Dense(self.par[i], self.par[i + 1], weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(self.par[i] + self.par[i + 1]), (self.par[i + 1], self.par[i])), dtype=mstype.float32), bias_init=Tensor(np.random.normal( 0.0, 1.0, (self.par[i + 1], )), dtype=mstype.float32))) self.fc = nn.CellList(fc) self.fc0 = deepcopy(self.fc) self.fc2 = [self.fc, self.fc0] fc = [] for i in range(3): fc.append( nn.Dense( self.neuron[i], self.neuron[i + 1], weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(self.neuron[i] + self.neuron[i + 1]), (self.neuron[i + 1], self.neuron[i])), dtype=mstype.float32), bias_init=Tensor(np.random.normal(0.0, 1.0, (self.neuron[i + 1], )), dtype=mstype.float32))) fc.append( nn.Dense(240, 1, weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(240 + 1), (1, 240)), dtype=mstype.float32), bias_init=Tensor(np.random.normal(type_bias_ae[0], 1.0, (1, )), dtype=mstype.float32))) for i in range(3): fc.append( nn.Dense( self.neuron[i], self.neuron[i + 1], weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(self.neuron[i] + self.neuron[i + 1]), (self.neuron[i + 1], self.neuron[i])), dtype=mstype.float32), bias_init=Tensor(np.random.normal(0.0, 1.0, (self.neuron[i + 1], )), dtype=mstype.float32))) fc.append( nn.Dense(240, 1, weight_init=Tensor(np.random.normal( 0.0, 1.0 / np.sqrt(240 + 1), (1, 240)), dtype=mstype.float32), bias_init=Tensor(np.random.normal(type_bias_ae[1], 1.0, (1, )), dtype=mstype.float32))) self.fc1 = nn.CellList(fc) xyz_A = np.vstack((np.identity(46), np.zeros([92, 46]))) self.xyz_A = Tensor(np.reshape(xyz_A, (1, 138, 46))) xyz_B = np.vstack((np.zeros([46, 92]), np.identity(92))) self.xyz_B = Tensor(np.reshape(xyz_B, (1, 138, 92))) xyz_2 = np.vstack( (np.identity(n_axis_neuron), np.zeros([self.par[-1] - n_axis_neuron, n_axis_neuron]))) self.xyz_2 = Tensor(xyz_2)
def __init__(self, transpose_x1, transpose_x2): super(Net, self).__init__() self.matmul = nn.MatMul(transpose_x1, transpose_x2)