def test_rnn( rnn_type="LSTM", input_size=500, hidden_size=50, sequence_length=10, batch_size=8, num_layers=2, bidirectional=True, lr=0.001, ): rnn_model = getattr(nn, rnn_type)(input_size, hidden_size, num_layers, bidirectional=bidirectional) num_directions = 2 if bidirectional else 1 model = PyTorchWrapperRNN(rnn_model) sgd = SGD(model.ops, lr) # Inputs and expected ouput X, Y = generate_data(sequence_length, batch_size, input_size, hidden_size, num_directions) # Test output shape check_Y_shape(model, X, Y, sequence_length, batch_size, hidden_size, num_directions) # Test initializing hidden initial_hidden = init_hidden(rnn_model, rnn_type, batch_size, num_layers * num_directions, hidden_size) # Test if sum of rnn output converges to with initial hidden check_learns_zero_output_rnn(model, sgd, X, Y, initial_hidden)
def test_update(): W = np.asarray([1.0, 0.0, 0.0, 1.0], dtype="f").reshape((2, 2)) bias = np.asarray([0.0, 0.0], dtype="f") model = Affine(2, 2) model.W[:] = W model.b[:] = bias sgd = SGD(model.ops, 1.0, L2=0.0) sgd.averages = None ff = np.asarray([[0.0, 0.0]], dtype="f") tf = np.asarray([[1.0, 0.0]], dtype="f") ft = np.asarray([[0.0, 1.0]], dtype="f") # noqa: F841 tt = np.asarray([[1.0, 1.0]], dtype="f") # noqa: F841 # ff, i.e. 0, 0 scores, finish_update = model.begin_update(ff) assert_allclose(scores[0, 0], scores[0, 1]) # Tell it the answer was 'f' gradient = np.asarray([[-1.0, 0.0]], dtype="f") finish_update(gradient, sgd) assert model.b[0] == 1.0 assert model.b[1] == 0.0 # Unchanged -- input was zeros, so can't get gradient for weights. assert model.W[0, 0] == 1.0 assert model.W[0, 1] == 0.0 assert model.W[1, 0] == 0.0 assert model.W[1, 1] == 1.0 # tf, i.e. 1, 0 scores, finish_update = model.begin_update(tf) # Tell it the answer was 'T' gradient = np.asarray([[0.0, -1.0]], dtype="f") finish_update(gradient, sgd) assert model.b[0] == 1.0 assert model.b[1] == 1.0 # Gradient for weights should have been outer(gradient, input) # so outer([0, -1.], [1., 0.]) # = [[0., 0.], [-1., 0.]] assert model.W[0, 0] == 1.0 - 0.0 assert model.W[0, 1] == 0.0 - 0.0 assert model.W[1, 0] == 0.0 - -1.0 assert model.W[1, 1] == 1.0 - 0.0
def test_unwrapped(nN=2, nI=3, nO=4): if PyTorchWrapper is None: return model = Affine(nO, nI) X = numpy.zeros((nN, nI), dtype="f") X += numpy.random.uniform(size=X.size).reshape(X.shape) sgd = SGD(model.ops, 0.001) Y = numpy.zeros((nN, nO), dtype="f") check_learns_zero_output(model, sgd, X, Y)
def test_wrapper(nN=2, nI=3, nO=4): if PyTorchWrapper is None: return model = PyTorchWrapper(torch.nn.Linear(nI, nO)) sgd = SGD(model.ops, 0.001) X = numpy.zeros((nN, nI), dtype="f") X += numpy.random.uniform(size=X.size).reshape(X.shape) Y = numpy.zeros((nN, nO), dtype="f") Yh, get_dX = model.begin_update(X) assert Yh.shape == (nN, nO) dYh = (Yh - Y) / Yh.shape[0] dX = get_dX(dYh, sgd=sgd) assert dX.shape == (nN, nI) check_learns_zero_output(model, sgd, X, Y)
def sgd(): return SGD(NumpyOps(), 0.001)