def __init__(self, config, is_training=True): super(BiLSTM, self).__init__() self.is_training = is_training self.batch_size = config.batch_size * config.rnn_batch_size print("batch size is {} ".format(self.batch_size)) self.input_size = config.input_size self.hidden_size = config.hidden_size self.num_step = config.num_step self.reshape = P.Reshape() self.cast = P.Cast() k = (1 / self.hidden_size) ** 0.5 self.rnn1 = P.DynamicRNN(forget_bias=0.0) self.rnn_bw = P.DynamicRNN(forget_bias=0.0) self.w1 = Parameter(np.random.uniform(-k, k, \ (self.input_size + self.hidden_size, 4 * self.hidden_size)).astype(np.float32), name="w1") self.w1_bw = Parameter(np.random.uniform(-k, k, \ (self.input_size + self.hidden_size, 4 * self.hidden_size)).astype(np.float32), name="w1_bw") self.b1 = Parameter(np.random.uniform(-k, k, (4 * self.hidden_size)).astype(np.float32), name="b1") self.b1_bw = Parameter(np.random.uniform(-k, k, (4 * self.hidden_size)).astype(np.float32), name="b1_bw") self.h1 = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float32)) self.h1_bw = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float32)) self.c1 = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float32)) self.c1_bw = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float32)) self.reverse_seq = P.ReverseV2(axis=[0]) self.concat = P.Concat() self.transpose = P.Transpose() self.concat1 = P.Concat(axis=2) self.dropout = nn.Dropout(0.7) self.use_dropout = config.use_dropout self.reshape = P.Reshape() self.transpose = P.Transpose()
def __init__(self, input_size, batch_size=64, hidden_size=512, num_class=11): super(StackedRNN, self).__init__() self.input_size = input_size self.batch_size = batch_size self.hidden_size = hidden_size self.num_class = num_class k = (1 / hidden_size) ** 0.5 self.rnn1 = P.DynamicRNN(forget_bias=0.0) self.rnn2 = P.DynamicRNN(forget_bias=0.0) self.w1 = Parameter(np.random.uniform(-k, k, (input_size + hidden_size, 4 * hidden_size)).astype(np.float16), name="w1") self.w2 = Parameter(np.random.uniform(-k, k, (hidden_size + hidden_size, 4 * hidden_size)).astype(np.float16), name="w2") self.b1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype(np.float16), name="b1") self.b2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype(np.float16), name="b2") self.h1 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.h2 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.c1 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.c2 = Tensor(np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.fc_weight = Tensor(np.random.random((hidden_size, num_class)).astype(np.float16)) self.fc_bias = Tensor(np.random.random(self.num_class).astype(np.float16)) self.reshape = P.Reshape() self.transpose = P.Transpose() self.matmul = nn.MatMul()
def __init__(self, config): super(CRNN, self).__init__() self.batch_size = config.batch_size self.input_size = config.input_size self.hidden_size = config.hidden_size self.num_classes = config.class_num self.reshape = P.Reshape() self.cast = P.Cast() k = (1 / self.hidden_size) ** 0.5 self.rnn1 = P.DynamicRNN(forget_bias=0.0) self.rnn1_bw = P.DynamicRNN(forget_bias=0.0) self.rnn2 = P.DynamicRNN(forget_bias=0.0) self.rnn2_bw = P.DynamicRNN(forget_bias=0.0) w1 = np.random.uniform(-k, k, (self.input_size + self.hidden_size, 4 * self.hidden_size)) self.w1 = Parameter(w1.astype(np.float16), name="w1") w2 = np.random.uniform(-k, k, (2 * self.hidden_size + self.hidden_size, 4 * self.hidden_size)) self.w2 = Parameter(w2.astype(np.float16), name="w2") w1_bw = np.random.uniform(-k, k, (self.input_size + self.hidden_size, 4 * self.hidden_size)) self.w1_bw = Parameter(w1_bw.astype(np.float16), name="w1_bw") w2_bw = np.random.uniform(-k, k, (2 * self.hidden_size + self.hidden_size, 4 * self.hidden_size)) self.w2_bw = Parameter(w2_bw.astype(np.float16), name="w2_bw") self.b1 = Parameter(np.random.uniform(-k, k, (4 * self.hidden_size)).astype(np.float16), name="b1") self.b2 = Parameter(np.random.uniform(-k, k, (4 * self.hidden_size)).astype(np.float16), name="b2") self.b1_bw = Parameter(np.random.uniform(-k, k, (4 * self.hidden_size)).astype(np.float16), name="b1_bw") self.b2_bw = Parameter(np.random.uniform(-k, k, (4 * self.hidden_size)).astype(np.float16), name="b2_bw") self.h1 = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.h2 = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.h1_bw = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.h2_bw = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.c1 = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.c2 = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.c1_bw = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.c2_bw = Tensor(np.zeros(shape=(1, self.batch_size, self.hidden_size)).astype(np.float16)) self.fc_weight = np.random.random((self.num_classes, self.hidden_size)).astype(np.float32) self.fc_bias = np.random.random((self.num_classes)).astype(np.float32) self.fc = nn.Dense(in_channels=self.hidden_size, out_channels=self.num_classes, weight_init=Tensor(self.fc_weight), bias_init=Tensor(self.fc_bias)) self.fc.to_float(mstype.float32) self.expand_dims = P.ExpandDims() self.concat = P.Concat() self.transpose = P.Transpose() self.squeeze = P.Squeeze(axis=0) self.vgg = VGG() self.reverse_seq1 = P.ReverseSequence(batch_dim=1, seq_dim=0) self.reverse_seq2 = P.ReverseSequence(batch_dim=1, seq_dim=0) self.reverse_seq3 = P.ReverseSequence(batch_dim=1, seq_dim=0) self.reverse_seq4 = P.ReverseSequence(batch_dim=1, seq_dim=0) self.seq_length = Tensor(np.ones((self.batch_size), np.int32) * config.num_step, mstype.int32) self.concat1 = P.Concat(axis=2) self.dropout = nn.Dropout(0.5) self.rnn_dropout = nn.Dropout(0.9) self.use_dropout = config.use_dropout
def __init__(self, input_size, batch_size=64, hidden_size=512): super(StackedRNN, self).__init__() self.batch_size = batch_size self.input_size = input_size self.num_classes = 11 self.reshape = P.Reshape() self.cast = P.Cast() k = (1 / hidden_size)**0.5 self.rnn1 = P.DynamicRNN(forget_bias=0.0) self.rnn2 = P.DynamicRNN(forget_bias=0.0) self.w1 = Parameter(np.random.uniform( -k, k, (input_size + hidden_size, 4 * hidden_size)).astype(np.float16), name="w1") self.w2 = Parameter(np.random.uniform( -k, k, (hidden_size + hidden_size, 4 * hidden_size)).astype(np.float16), name="w2") self.b1 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype( np.float16), name="b1") self.b2 = Parameter(np.random.uniform(-k, k, (4 * hidden_size)).astype( np.float16), name="b2") self.h1 = Tensor( np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.h2 = Tensor( np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.c1 = Tensor( np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.c2 = Tensor( np.zeros(shape=(1, batch_size, hidden_size)).astype(np.float16)) self.fc_weight = np.random.random( (self.num_classes, hidden_size)).astype(np.float32) self.fc_bias = np.random.random(self.num_classes).astype(np.float32) self.fc = nn.Dense(in_channels=hidden_size, out_channels=self.num_classes, weight_init=Tensor(self.fc_weight), bias_init=Tensor(self.fc_bias)) self.fc.to_float(mstype.float32) self.expand_dims = P.ExpandDims() self.concat = P.Concat() self.transpose = P.Transpose() self.squeeze = P.Squeeze(axis=0)
def __init__(self, num_setp=50, batch_size=128, word_embed_dim=1024, hidden_size=1024, initializer_range=0.1): super(DynamicRNNCell, self).__init__() self.rnn = P.DynamicRNN() self.num_step = num_setp self.batch_size = batch_size self.input_size = word_embed_dim self.hidden_size = hidden_size # w dynamicRNN_w = np.random.uniform( -initializer_range, initializer_range, size=[self.input_size + self.hidden_size, 4 * self.hidden_size]) self.dynamicRNN_w = Parameter(Tensor(dynamicRNN_w, mstype.float32)) # b dynamicRNN_b = np.random.uniform(-initializer_range, initializer_range, size=[4 * self.hidden_size]) self.dynamicRNN_b = Parameter(Tensor(dynamicRNN_b, mstype.float32)) self.dynamicRNN_h = Tensor( np.zeros((1, self.batch_size, self.hidden_size)), mstype.float32) self.dynamicRNN_c = Tensor( np.zeros((1, self.batch_size, self.hidden_size)), mstype.float32) self.cast = P.Cast()
def __init__(self, input_size, hidden_size, num_layers=1, has_bias=True, batch_first=False, dropout=0, bidirectional=False): super(LSTM, self).__init__() validator.check_value_type("batch_first", batch_first, [bool], self.cls_name) validator.check_positive_int(hidden_size, "hidden_size", self.cls_name) validator.check_positive_int(num_layers, "num_layers", self.cls_name) self.is_ascend = context.get_context("device_target") == "Ascend" self.batch_first = batch_first self.transpose = P.Transpose() self.num_layers = num_layers self.bidirectional = bidirectional self.dropout = dropout self.lstm = P.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, has_bias=has_bias, bidirectional=bidirectional, dropout=float(dropout)) weight_size = 0 gate_size = 4 * hidden_size stdv = 1 / math.sqrt(hidden_size) num_directions = 2 if bidirectional else 1 if self.is_ascend: self.reverse_seq = P.ReverseSequence(batch_dim=1, seq_dim=0) self.concat = P.Concat(axis=0) self.concat_2dim = P.Concat(axis=2) self.cast = P.Cast() self.shape = P.Shape() if dropout != 0: self.dropout_op = nn.Dropout(float(dropout)) b0 = np.zeros(gate_size, dtype=np.float16) self.w_list = [] self.b_list = [] self.rnns_fw = P.DynamicRNN(forget_bias=0.0) self.rnns_bw = P.DynamicRNN(forget_bias=0.0) for layer in range(num_layers): w_shape = input_size if layer == 0 else (num_directions * hidden_size) w_np = np.random.uniform( -stdv, stdv, (w_shape + hidden_size, gate_size)).astype(np.float16) self.w_list.append( Parameter(initializer(Tensor(w_np), [w_shape + hidden_size, gate_size]), name='weight_fw' + str(layer))) if has_bias: b_np = np.random.uniform(-stdv, stdv, gate_size).astype(np.float16) self.b_list.append( Parameter(initializer(Tensor(b_np), [gate_size]), name='bias_fw' + str(layer))) else: self.b_list.append( Parameter(initializer(Tensor(b0), [gate_size]), name='bias_fw' + str(layer))) if bidirectional: w_bw_np = np.random.uniform( -stdv, stdv, (w_shape + hidden_size, gate_size)).astype(np.float16) self.w_list.append( Parameter( initializer(Tensor(w_bw_np), [w_shape + hidden_size, gate_size]), name='weight_bw' + str(layer))) b_bw_np = np.random.uniform( -stdv, stdv, (4 * hidden_size)).astype(np.float16) if has_bias else b0 self.b_list.append( Parameter(initializer(Tensor(b_bw_np), [gate_size]), name='bias_bw' + str(layer))) self.w_list = ParameterTuple(self.w_list) self.b_list = ParameterTuple(self.b_list) else: for layer in range(num_layers): input_layer_size = input_size if layer == 0 else hidden_size * num_directions increment_size = gate_size * input_layer_size increment_size += gate_size * hidden_size if has_bias: increment_size += 2 * gate_size weight_size += increment_size * num_directions w_np = np.random.uniform(-stdv, stdv, (weight_size, 1, 1)).astype(np.float32) self.weight = Parameter(initializer(Tensor(w_np), [weight_size, 1, 1]), name='weight')
def __init__(self, weight, vocab_size, cell, batch_size): super(textrcnn, self).__init__() self.num_hiddens = 512 self.embed_size = 300 self.num_classes = 2 self.batch_size = batch_size k = (1 / self.num_hiddens)**0.5 self.embedding = nn.Embedding(vocab_size, self.embed_size, embedding_table=weight) self.embedding.embedding_table.requires_grad = False self.cell = cell self.cast = P.Cast() self.h1 = Tensor( np.zeros(shape=(self.batch_size, self.num_hiddens)).astype(np.float16)) self.c1 = Tensor( np.zeros(shape=(self.batch_size, self.num_hiddens)).astype(np.float16)) if cell == "lstm": self.lstm = P.DynamicRNN(forget_bias=0.0) self.w1_fw = Parameter(np.random.uniform( -k, k, (self.embed_size + self.num_hiddens, 4 * self.num_hiddens)).astype(np.float16), name="w1_fw") self.b1_fw = Parameter(np.random.uniform( -k, k, (4 * self.num_hiddens)).astype(np.float16), name="b1_fw") self.w1_bw = Parameter(np.random.uniform( -k, k, (self.embed_size + self.num_hiddens, 4 * self.num_hiddens)).astype(np.float16), name="w1_bw") self.b1_bw = Parameter(np.random.uniform( -k, k, (4 * self.num_hiddens)).astype(np.float16), name="b1_bw") self.h1 = Tensor( np.zeros(shape=(1, self.batch_size, self.num_hiddens)).astype(np.float16)) self.c1 = Tensor( np.zeros(shape=(1, self.batch_size, self.num_hiddens)).astype(np.float16)) if cell == "vanilla": self.rnnW_fw = nn.Dense(self.num_hiddens, self.num_hiddens) self.rnnU_fw = nn.Dense(self.embed_size, self.num_hiddens) self.rnnW_bw = nn.Dense(self.num_hiddens, self.num_hiddens) self.rnnU_bw = nn.Dense(self.embed_size, self.num_hiddens) if cell == "gru": self.rnnWr_fw = nn.Dense(self.num_hiddens + self.embed_size, self.num_hiddens) self.rnnWz_fw = nn.Dense(self.num_hiddens + self.embed_size, self.num_hiddens) self.rnnWh_fw = nn.Dense(self.num_hiddens + self.embed_size, self.num_hiddens) self.rnnWr_bw = nn.Dense(self.num_hiddens + self.embed_size, self.num_hiddens) self.rnnWz_bw = nn.Dense(self.num_hiddens + self.embed_size, self.num_hiddens) self.rnnWh_bw = nn.Dense(self.num_hiddens + self.embed_size, self.num_hiddens) self.ones = Tensor( np.ones(shape=(self.batch_size, self.num_hiddens)).astype(np.float16)) self.rnnWr_fw.to_float(mstype.float16) self.rnnWz_fw.to_float(mstype.float16) self.rnnWh_fw.to_float(mstype.float16) self.rnnWr_bw.to_float(mstype.float16) self.rnnWz_bw.to_float(mstype.float16) self.rnnWh_bw.to_float(mstype.float16) self.transpose = P.Transpose() self.reduce_max = P.ReduceMax() self.expand_dims = P.ExpandDims() self.concat = P.Concat() self.reshape = P.Reshape() self.left_pad_tensor = Tensor( np.zeros( (1, self.batch_size, self.num_hiddens)).astype(np.float16)) self.right_pad_tensor = Tensor( np.zeros( (1, self.batch_size, self.num_hiddens)).astype(np.float16)) self.output_dense = nn.Dense(self.num_hiddens * 1, 2) self.concat0 = P.Concat(0) self.concat2 = P.Concat(2) self.concat1 = P.Concat(1) self.text_rep_dense = nn.Dense(2 * self.num_hiddens + self.embed_size, self.num_hiddens) self.mydense = nn.Dense(self.num_hiddens, 2) self.drop_out = nn.Dropout(keep_prob=0.7) self.tanh = P.Tanh() self.sigmoid = P.Sigmoid() self.slice = P.Slice() self.text_rep_dense.to_float(mstype.float16) self.mydense.to_float(mstype.float16) self.output_dense.to_float(mstype.float16)
def __init__(self, bidirectional=False): super(LSTM_Ascend, self).__init__() self.bidirectional = bidirectional self.dynamic_rnn = P.DynamicRNN(forget_bias=0.0) self.reverseV2 = P.ReverseV2(axis=[0]) self.concat = P.Concat(2)