def test_vanillaRNN_gpu_tiny_ops_shape_check(self): # gradients shape check. inputs, target, h0 = prepare_inputs_targets_for_rnn_test() rnn = autograd.RNN(3, 2) hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) # d=autograd.infer_dependency(loss.creator) # print(d) for t, dt in autograd.backward(loss): self.check_shape(t.shape, dt.shape)
def test_numerical_gradients_check_for_vallina_rnn(self): inputs, target, h0 = prepare_inputs_targets_for_rnn_test() rnn = autograd.RNN(3, 2) def valinna_rnn_forward(): hs, _ = rnn(inputs, h0) loss = autograd.softmax_cross_entropy(hs[0], target[0]) for i in range(1, len(hs)): l = autograd.softmax_cross_entropy(hs[i], target[i]) loss = autograd.add(loss, l) #grads = autograd.gradients(loss) return loss loss1 = valinna_rnn_forward() auto_grads = autograd.gradients(loss1) for param in rnn.params: auto_grad = tensor.to_numpy(auto_grads[param]) self.gradients_check(valinna_rnn_forward, param, auto_grad)
from singa import autograd from singa import optimizer from singa import * from singa import sonnx import onnx from singa import device import numpy as np autograd.training = True np.random.seed(0) data = np.random.randn(4, 2).astype(np.float32) h = np.random.randn(4, 2).astype(np.float32) label = np.random.randn(4, 2).astype(np.float32) print('train_data_shape:', data.shape) print('train_label_shape:', label.shape) rnn = autograd.RNN(2, 2) sgd = optimizer.SGD(0.00) dev = device.get_default_device() # training process for i in range(1): inputs = tensor.Tensor(device=dev, data=data, stores_grad=False) h0 = tensor.Tensor(device=dev, data=h, stores_grad=False) targets = tensor.Tensor(device=dev, data=label, requires_grad=False, stores_grad=False) x = rnn(inputs, h0) loss = autograd.mse_loss(x, targets) gradient = autograd.backward(loss) for p, gp in gradient: