def test_rnn(
rnn_type="LSTM",
input_size=500,
hidden_size=50,
sequence_length=10,
batch_size=8,
num_layers=2,
bidirectional=True,
lr=0.001,
):
rnn_model = getattr(nn, rnn_type)(input_size,
hidden_size,
num_layers,
bidirectional=bidirectional)
num_directions = 2 if bidirectional else 1
model = PyTorchWrapperRNN(rnn_model)
sgd = SGD(model.ops, lr)
# Inputs and expected ouput
X, Y = generate_data(sequence_length, batch_size, input_size, hidden_size,
num_directions)
# Test output shape
check_Y_shape(model, X, Y, sequence_length, batch_size, hidden_size,
num_directions)
# Test initializing hidden
initial_hidden = init_hidden(rnn_model, rnn_type, batch_size,
num_layers * num_directions, hidden_size)
# Test if sum of rnn output converges to with initial hidden
check_learns_zero_output_rnn(model, sgd, X, Y, initial_hidden)
def test_update():
W = np.asarray([1.0, 0.0, 0.0, 1.0], dtype="f").reshape((2, 2))
bias = np.asarray([0.0, 0.0], dtype="f")
model = Affine(2, 2)
model.W[:] = W
model.b[:] = bias
sgd = SGD(model.ops, 1.0, L2=0.0)
sgd.averages = None
ff = np.asarray([[0.0, 0.0]], dtype="f")
tf = np.asarray([[1.0, 0.0]], dtype="f")
ft = np.asarray([[0.0, 1.0]], dtype="f") # noqa: F841
tt = np.asarray([[1.0, 1.0]], dtype="f") # noqa: F841
# ff, i.e. 0, 0
scores, finish_update = model.begin_update(ff)
assert_allclose(scores[0, 0], scores[0, 1])
# Tell it the answer was 'f'
gradient = np.asarray([[-1.0, 0.0]], dtype="f")
finish_update(gradient, sgd)
assert model.b[0] == 1.0
assert model.b[1] == 0.0
# Unchanged -- input was zeros, so can't get gradient for weights.
assert model.W[0, 0] == 1.0
assert model.W[0, 1] == 0.0
assert model.W[1, 0] == 0.0
assert model.W[1, 1] == 1.0
# tf, i.e. 1, 0
scores, finish_update = model.begin_update(tf)
# Tell it the answer was 'T'
gradient = np.asarray([[0.0, -1.0]], dtype="f")
finish_update(gradient, sgd)
assert model.b[0] == 1.0
assert model.b[1] == 1.0
# Gradient for weights should have been outer(gradient, input)
# so outer([0, -1.], [1., 0.])
# = [[0., 0.], [-1., 0.]]
assert model.W[0, 0] == 1.0 - 0.0
assert model.W[0, 1] == 0.0 - 0.0
assert model.W[1, 0] == 0.0 - -1.0
assert model.W[1, 1] == 1.0 - 0.0
def test_unwrapped(nN=2, nI=3, nO=4):
if PyTorchWrapper is None:
return
model = Affine(nO, nI)
X = numpy.zeros((nN, nI), dtype="f")
X += numpy.random.uniform(size=X.size).reshape(X.shape)
sgd = SGD(model.ops, 0.001)
Y = numpy.zeros((nN, nO), dtype="f")
check_learns_zero_output(model, sgd, X, Y)
def test_wrapper(nN=2, nI=3, nO=4):
if PyTorchWrapper is None:
return
model = PyTorchWrapper(torch.nn.Linear(nI, nO))
sgd = SGD(model.ops, 0.001)
X = numpy.zeros((nN, nI), dtype="f")
X += numpy.random.uniform(size=X.size).reshape(X.shape)
Y = numpy.zeros((nN, nO), dtype="f")
Yh, get_dX = model.begin_update(X)
assert Yh.shape == (nN, nO)
dYh = (Yh - Y) / Yh.shape[0]
dX = get_dX(dYh, sgd=sgd)
assert dX.shape == (nN, nI)
check_learns_zero_output(model, sgd, X, Y)
def sgd():
return SGD(NumpyOps(), 0.001)