def test_maxpooling(self):
X = np.random.randn(1, 1, 6, 6)
tX = val(X)
tY = rd.build_max_pooling(tX, 2, 2, 2, 2)
tYf = rd.build_reshape(tY, (-1, ))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dY = torch.nn.functional.max_pool2d(dX, (2, 2), (2, 2))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tX).eval(), dX.grad.data.numpy())
X = np.random.randn(1, 1, 5, 5)
tX = val(X)
tY = rd.build_max_pooling(tX, 2, 2, 1, 1)
tYf = rd.build_reshape(tY, (-1, ))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dY = torch.nn.functional.max_pool2d(dX, (2, 2), (1, 1))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tX).eval(), dX.grad.data.numpy())
X = np.random.randn(2, 3, 9, 7)
tX = val(X)
tY = rd.build_max_pooling(tX, 3, 2, 2, 1)
tYf = rd.build_reshape(tY, (-1, ))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dY = torch.nn.functional.max_pool2d(dX, (3, 2), (2, 1))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tX).eval(), dX.grad.data.numpy())
def test_fwd_dotvv(self):
x = np.random.randn(7)
y = np.random.randn(7)
tx = val(x)
ty = val(y)
tz = rd.build_dot_vv(tx, ty)
self.ck_fequals(tz.eval(), x @ y)
def test_bwd_dotvv(self):
x = np.random.randn(7)
y = np.random.randn(7)
tx = val(x)
ty = val(y)
tlone = val(10)
tz = rd.build_dot_vv(tx, ty)
self.ck_fequals(get_grad(tz, tx).eval(), y)
self.ck_fequals(get_grad(tz, ty).eval(), x)
self.ck_fequals(get_grad(tz, tlone).eval(), 0)
def test_bwd_vlog(self):
x = np.random.rand(12).astype(np.float32) + 0.1
tx = val(x)
tz = rd.build_vlog(tx)
te = rd.build_dot_vv(tz, tz)
dx = torch.tensor(x, requires_grad=True)
dz = torch.log(dx)
de = torch.dot(dz, dz)
de.backward()
self.ck_fequals(get_grad(te, tx).eval(), dx.grad.data.numpy())
def test_act_elu(self):
x = np.random.randn(43) * 4
tx = val(x)
tz = rd.build_velu(tx, 0.05)
te = rd.build_dot_vv(tz, tz)
dx = torch.tensor(x, requires_grad=True)
dz = torch.nn.functional.elu(dx, 0.05)
de = torch.dot(dz, dz)
de.backward()
self.ck_fequals(tz.eval(), dz.data.numpy())
self.ck_fequals(get_grad(te, tx).eval(), dx.grad.data.numpy())
def test_act_sigmoid(self):
x = np.random.randn(23)
tx = val(x)
tz = rd.build_vsigmoid(tx)
te = rd.build_dot_vv(tz, tz)
dx = torch.tensor(x, requires_grad=True)
dz = torch.sigmoid(dx)
de = torch.dot(dz, dz)
de.backward()
self.ck_fequals(tz.eval(), dz.data.numpy())
self.ck_fequals(get_grad(te, tx).eval(), dx.grad.data.numpy())
def test_act_softmax(self):
x = np.random.randn(11, 7)
tx = val(x)
tz = rd.build_softmax(tx)
tz = rd.build_reshape(tz, (11 * 7, ))
te = rd.build_dot_vv(tz, tz)
dx = torch.tensor(x, requires_grad=True)
dz = torch.relu(dx).view(-1)
de = torch.dot(dz, dz)
de.backward()
self.ck_fequals(tz.eval(), dz.data.numpy(), feps=1e-1)
def test_sgd_logreg_k_l1_l2(self):
X = np.random.randn(46, 7).astype(np.float32)
w = np.random.randn(7, 4).astype(np.float32)
y_true = np.zeros((46, 4)).astype(np.float32)
for i in range(y_true.shape[0]):
y_true[i][np.random.randint(0, y_true.shape[1])] = 1
alpha_l1 = 0.53
alpha_l2 = 0.82
dX = val(X)
dw = val(w)
dw_flat = rd.build_reshape(dw, (dw.shape[0] * dw.shape[1], ))
dy_true = val(y_true)
dy_out = rd.build_dot_mm(dX, dw)
dy_pred = rd.build_softmax(dy_out)
dloss = rd.build_cross_entropy_loss(dy_out, dy_true)
dloss = dloss + alpha_l1 * rd.build_norm1(dw_flat)
dloss = dloss + alpha_l2 * rd.build_dot_vv(dw_flat, dw_flat)
tX = torch.tensor(X, requires_grad=True)
tw = torch.tensor(w, requires_grad=True)
tw_flat = tw.view(-1)
ty_true = torch.tensor(y_true, requires_grad=False)
ty_true = torch.argmax(ty_true, dim=1)
ty_out = torch.matmul(tX, tw)
ty_pred = torch.nn.functional.softmax(ty_out, dim=1)
utils.save_grad(ty_out)
criterion = torch.nn.CrossEntropyLoss(reduction='sum')
tloss = criterion(ty_out, ty_true)
tloss = tloss + alpha_l1 * torch.norm(
tw_flat, p=1) + alpha_l2 * torch.dot(tw_flat, tw_flat)
tloss.backward()
self.ck_fequals(dloss.eval(), tloss.data.numpy(), feps=1e-3)
self.ck_fequals(dy_pred.eval(), ty_pred.data.numpy())
self.ck_fequals(
get_grad(dloss, dy_out).eval(),
utils.get_grad(ty_out).data.numpy())
self.ck_fequals(get_grad(dloss, dw).eval(), tw.grad.data.numpy())
self.ck_fequals(get_grad(dloss, dX).eval(), tX.grad.data.numpy())
def test_bwd_vsdiv(self):
x = np.random.randn()
y = np.random.rand(12).astype(np.float32) + 0.1
tx = val(x)
ty = val(y)
tz = rd.build_vsdiv(tx, ty)
te = rd.build_dot_vv(tz, tz)
dx = torch.tensor(x, requires_grad=True)
dy = torch.tensor(y, requires_grad=True)
dz = dx / dy
de = torch.dot(dz, dz)
de.backward()
self.ck_fequals(get_grad(te, tx).eval(),
dx.grad.data.numpy(),
feps=1e-4)
self.ck_fequals(get_grad(te, ty).eval(), dy.grad.data.numpy())
def test_conv2d_transpose(self):
X = np.random.randn(2, 4, 13, 16).astype(np.float32)
K = np.random.randn(4, 3, 5, 8).astype(np.float32)
b = np.random.randn(3).astype(np.float32)
tX = val(X)
tK = val(K)
tb = val(b)
tY = rd.op_conv2d_transpose(tX, tK, 1, 1, 0, 0)
tY = rd.build_conv2d_bias_add(tY, tb)
tYf = rd.build_reshape(tY, ((-1, )))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dK = torch.tensor(K, requires_grad=True)
db = torch.tensor(b, requires_grad=True)
dY = torch.nn.functional.conv_transpose2d(dX,
dK,
bias=db,
stride=(1, 1))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tK).eval(),
dK.grad.data.numpy(),
feps=1e-4)
self.ck_fequals(get_grad(te, tX).eval(),
dX.grad.data.numpy(),
feps=1e-5)
self.ck_fequals(get_grad(te, tb).eval(),
db.grad.data.numpy(),
feps=1e-3)
X = np.random.randn(2, 4, 5, 5).astype(np.float32)
K = np.random.randn(4, 3, 5, 8).astype(np.float32)
tX = val(X)
tK = val(K)
tY = rd.op_conv2d_transpose(tX, tK, 3, 4, 0, 0)
tYf = rd.build_reshape(tY, ((-1, )))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dK = torch.tensor(K, requires_grad=True)
dY = torch.nn.functional.conv_transpose2d(dX, dK, stride=(3, 4))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tK).eval(),
dK.grad.data.numpy(),
feps=1e-5)
self.ck_fequals(get_grad(te, tX).eval(),
dX.grad.data.numpy(),
feps=1e-5)
X = np.random.randn(2, 4, 5, 5).astype(np.float32)
K = np.random.randn(4, 3, 5, 8).astype(np.float32)
tX = val(X)
tK = val(K)
tY = rd.op_conv2d_transpose(tX, tK, 3, 4, 6, 8)
tYf = rd.build_reshape(tY, ((-1, )))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dK = torch.tensor(K, requires_grad=True)
dY = torch.nn.functional.conv_transpose2d(dX,
dK,
stride=(3, 4),
padding=(6, 8))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tK).eval(),
dK.grad.data.numpy(),
feps=1e-5)
self.ck_fequals(get_grad(te, tX).eval(),
dX.grad.data.numpy(),
feps=1e-5)
X = np.random.randn(2, 4, 9, 11).astype(np.float32)
K = np.random.randn(4, 3, 6, 8).astype(np.float32)
tX = val(X)
tK = val(K)
tY = rd.op_conv2d_transpose(tX, tK, 3, 4, 7, 11)
tYf = rd.build_reshape(tY, ((-1, )))
te = rd.build_dot_vv(tYf, tYf)
dX = torch.tensor(X, requires_grad=True)
dK = torch.tensor(K, requires_grad=True)
dY = torch.nn.functional.conv_transpose2d(dX,
dK,
stride=(3, 4),
padding=(7, 11))
dYf = dY.view(-1)
de = torch.dot(dYf, dYf)
de.backward()
self.ck_fequals(tY.eval(), dY.data.numpy())
self.ck_fequals(get_grad(te, tK).eval(),
dK.grad.data.numpy(),
feps=1e-5)
self.ck_fequals(get_grad(te, tX).eval(),
dX.grad.data.numpy(),
feps=1e-5)
