def lstm_forward(): hs, _, _ = rnn(inputs, (h0, c0)) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) return loss
def valinna_rnn_forward(): hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) #grads = autograd.gradients(loss) return loss
def __call__(self, x): y = self.layers[0](x) for layer in self.layers[1:]: if isinstance(y, tuple): y = y[0] y = layer(y) if self.skip is not None: skip = self.skip(x) skip = self.skipbn(skip) else: skip = x y = autograd.add(y, skip) return y
def test_vanillaRNN_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() rnn = autograd.RNN(3, 2) hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def __call__(self, x): y = self.layers[0](x) for layer in self.layers[1:]: if isinstance(y, tuple): y = y[0] y = layer(y) if self.skip is not None: skip = self.skip(x) skip = self.skipbn(skip) else: skip = x # print(skip.shape) #print(y.shape, skip.shape) y = autograd.add(y, skip) return y
def __call__(self, x): residual = x out = self.conv1(x) out = self.bn1(out) out = autograd.relu(out) out = self.conv2(out) out = self.bn2(out) if self.downsample is not None: residual = self.downsample(x) out = autograd.add(out, residual) out = autograd.relu(out) return out
def test_LSTM_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() c_0 = np.random.random((2, 1)).astype(np.float32) c0 = tensor.Tensor(device=gpu_dev, data=c_0) rnn = autograd.LSTM(3, 2) hs, _, _ = rnn(inputs, (h0, c0)) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def run(model, modeldic, layer,inputs): ''' input: input for singa model load other nodes of onnx ''' supportLayer = ['Linear','Conv','MaxPool','AveragePool','BatchNormalization'] #supportLayer = ['Conv', 'MaxPool', 'AveragePool', 'BatchNormalization'] oper=modeldic for counter,i in enumerate(model.graph.input): oper[i.name] = inputs[counter] for i in model.graph.node: if (i.op_type == 'Relu'): oper[str(i.output[0])] = autograd.relu(oper[str(i.input[0])]) elif (i.op_type == 'Softmax'): oper[str(i.output[0])] = autograd.softmax(oper[str(i.input[0])]) elif (i.op_type == 'Add'): oper[str(i.output[0])] = autograd.add(oper[str(i.input[0])], oper[str(i.input[1])]) elif (i.op_type == 'MatMul'): oper[str(i.output[0])] = autograd.matmul(oper[str(i.input[0])], oper[str(i.input[1])]) elif (i.op_type == 'Flatten'): oper[str(i.output[0])] = autograd.flatten(oper[str(i.input[0])]) elif(i.op_type == 'Concat'): oper[str(i.output[0])] = autograd.cat((oper[str(i.input[0])], oper[str(i.input[1])]),int(i.attribute[0].i)) elif(i.op_type == 'Tanh'): oper[str(i.output[0])] = autograd.tanh(oper[str(i.input[0])]) elif (i.op_type == 'Sigmoid'): oper[str(i.output[0])] = autograd.sigmoid(oper[str(i.input[0])]) elif (i.op_type == 'Mul'): oper[str(i.output[0])] = autograd.mul(oper[str(i.input[0])],oper[str(i.input[1])]) elif (i.op_type in supportLayer): oper[str(i.output[0])] = layer[str(i.output[0])](oper[str(i.input[0])]) out =[] for counter,i in enumerate(model.graph.output): out.append(modeldic[i.name]) return out
label = to_categorical(label, 2).astype(np.float32) print('train_data_shape:', data.shape) print('train_label_shape:', label.shape) inputs = Tensor(data=data) target = Tensor(data=label) linear1 = autograd.Linear(3, 2) linear2 = autograd.Linear(2, 2) linear3 = autograd.Linear(2, 2) sgd = optimizer.SGD(0.00) # training process for i in range(1): x = linear1(inputs) x = autograd.relu(x) x1 = linear2(x) x2 = linear3(x) x3 = autograd.add(x1, x2) y = autograd.softmax(x3) loss = autograd.cross_entropy(y, target) gradient = autograd.backward(loss) for p, gp in gradient: sgd.apply(0, gp, p, '') if (i % 100 == 0): print('training loss = ', tensor.to_numpy(loss)[0]) model = sonnx.to_onnx_model([inputs], [y]) onnx.save(model, 'linear.onnx')