Beispiel #1
0
    def test_layer_lin1(self):

        X = np.random.randn(46, 7)
        y_true = np.random.randn(46, 3)
        W = np.random.randn(7, 3)
        b = np.random.randn(3)

        dX = val(X)
        dy_true = val(y_true)
        dW = val(W)
        db = val(b)
        dy_pred = rd.build_add_bias(rd.build_dot_mm(dX, dW), db)
        dloss = mse(rd.build_reshape(dy_pred, (y_true.size, )),
                    rd.build_reshape(dy_true, (y_true.size, )))

        tX = torch.tensor(X, requires_grad=True)
        ty_true = torch.tensor(y_true, requires_grad=True)
        tW = torch.tensor(W, requires_grad=True)
        tb = torch.tensor(b, requires_grad=True)
        ty_pred = torch.matmul(tX, tW) + tb
        criterion = torch.nn.MSELoss()
        tloss = criterion(ty_pred, ty_true)
        tloss.backward()

        self.ck_fequals(dloss.eval(), tloss.data.numpy(), feps=1e-3)
        self.ck_fequals(
            get_grad(dloss, dy_true).eval(), ty_true.grad.data.numpy())
        self.ck_fequals(get_grad(dloss, dW).eval(),
                        tW.grad.data.numpy(),
                        feps=1e-5)
        self.ck_fequals(get_grad(dloss, db).eval(),
                        tb.grad.data.numpy(),
                        feps=1e-5)
        self.ck_fequals(get_grad(dloss, dX).eval(), tX.grad.data.numpy())
Beispiel #2
0
    def test_maxpooling(self):

        X = np.random.randn(1, 1, 6, 6)

        tX = val(X)
        tY = rd.build_max_pooling(tX, 2, 2, 2, 2)
        tYf = rd.build_reshape(tY, (-1, ))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dY = torch.nn.functional.max_pool2d(dX, (2, 2), (2, 2))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tX).eval(), dX.grad.data.numpy())

        X = np.random.randn(1, 1, 5, 5)

        tX = val(X)
        tY = rd.build_max_pooling(tX, 2, 2, 1, 1)
        tYf = rd.build_reshape(tY, (-1, ))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dY = torch.nn.functional.max_pool2d(dX, (2, 2), (1, 1))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tX).eval(), dX.grad.data.numpy())

        X = np.random.randn(2, 3, 9, 7)

        tX = val(X)
        tY = rd.build_max_pooling(tX, 3, 2, 2, 1)
        tYf = rd.build_reshape(tY, (-1, ))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dY = torch.nn.functional.max_pool2d(dX, (3, 2), (2, 1))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tX).eval(), dX.grad.data.numpy())
Beispiel #3
0
            def forward(self, x):
                x = rd.build_vrelu(self.conv1(x))
                x = rd.build_max_pooling(x, 2, 2, 2, 2)

                x = rd.build_vrelu(self.conv2(x))
                x = rd.build_max_pooling(x, 2, 2, 2, 2)

                x = rd.build_reshape(x, (x.shape[0], -1))
                x = self.fc(x)
                return x
Beispiel #4
0
    def test_act_softmax(self):

        x = np.random.randn(11, 7)

        tx = val(x)
        tz = rd.build_softmax(tx)
        tz = rd.build_reshape(tz, (11 * 7, ))
        te = rd.build_dot_vv(tz, tz)

        dx = torch.tensor(x, requires_grad=True)
        dz = torch.relu(dx).view(-1)
        de = torch.dot(dz, dz)
        de.backward()

        self.ck_fequals(tz.eval(), dz.data.numpy(), feps=1e-1)
Beispiel #5
0
    def test_sgd_logreg_k_l1_l2(self):
        X = np.random.randn(46, 7).astype(np.float32)
        w = np.random.randn(7, 4).astype(np.float32)
        y_true = np.zeros((46, 4)).astype(np.float32)
        for i in range(y_true.shape[0]):
            y_true[i][np.random.randint(0, y_true.shape[1])] = 1

        alpha_l1 = 0.53
        alpha_l2 = 0.82

        dX = val(X)
        dw = val(w)
        dw_flat = rd.build_reshape(dw, (dw.shape[0] * dw.shape[1], ))
        dy_true = val(y_true)
        dy_out = rd.build_dot_mm(dX, dw)
        dy_pred = rd.build_softmax(dy_out)
        dloss = rd.build_cross_entropy_loss(dy_out, dy_true)
        dloss = dloss + alpha_l1 * rd.build_norm1(dw_flat)
        dloss = dloss + alpha_l2 * rd.build_dot_vv(dw_flat, dw_flat)

        tX = torch.tensor(X, requires_grad=True)
        tw = torch.tensor(w, requires_grad=True)
        tw_flat = tw.view(-1)
        ty_true = torch.tensor(y_true, requires_grad=False)
        ty_true = torch.argmax(ty_true, dim=1)
        ty_out = torch.matmul(tX, tw)
        ty_pred = torch.nn.functional.softmax(ty_out, dim=1)
        utils.save_grad(ty_out)
        criterion = torch.nn.CrossEntropyLoss(reduction='sum')
        tloss = criterion(ty_out, ty_true)
        tloss = tloss + alpha_l1 * torch.norm(
            tw_flat, p=1) + alpha_l2 * torch.dot(tw_flat, tw_flat)

        tloss.backward()

        self.ck_fequals(dloss.eval(), tloss.data.numpy(), feps=1e-3)
        self.ck_fequals(dy_pred.eval(), ty_pred.data.numpy())
        self.ck_fequals(
            get_grad(dloss, dy_out).eval(),
            utils.get_grad(ty_out).data.numpy())
        self.ck_fequals(get_grad(dloss, dw).eval(), tw.grad.data.numpy())
        self.ck_fequals(get_grad(dloss, dX).eval(), tX.grad.data.numpy())
Beispiel #6
0
 def forward(self, x):
     x = rd.build_reshape(x, (-1, IN_SIZE))
     x = rd.build_vrelu(self.l1(x))
     x = rd.build_vrelu(self.l2(x))
     y_logits = self.l3(x)
     return y_logits
Beispiel #7
0
    def test_conv2d_transpose(self):

        X = np.random.randn(2, 4, 13, 16).astype(np.float32)
        K = np.random.randn(4, 3, 5, 8).astype(np.float32)
        b = np.random.randn(3).astype(np.float32)

        tX = val(X)
        tK = val(K)
        tb = val(b)
        tY = rd.op_conv2d_transpose(tX, tK, 1, 1, 0, 0)
        tY = rd.build_conv2d_bias_add(tY, tb)
        tYf = rd.build_reshape(tY, ((-1, )))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dK = torch.tensor(K, requires_grad=True)
        db = torch.tensor(b, requires_grad=True)
        dY = torch.nn.functional.conv_transpose2d(dX,
                                                  dK,
                                                  bias=db,
                                                  stride=(1, 1))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tK).eval(),
                        dK.grad.data.numpy(),
                        feps=1e-4)
        self.ck_fequals(get_grad(te, tX).eval(),
                        dX.grad.data.numpy(),
                        feps=1e-5)
        self.ck_fequals(get_grad(te, tb).eval(),
                        db.grad.data.numpy(),
                        feps=1e-3)

        X = np.random.randn(2, 4, 5, 5).astype(np.float32)
        K = np.random.randn(4, 3, 5, 8).astype(np.float32)

        tX = val(X)
        tK = val(K)
        tY = rd.op_conv2d_transpose(tX, tK, 3, 4, 0, 0)
        tYf = rd.build_reshape(tY, ((-1, )))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dK = torch.tensor(K, requires_grad=True)
        dY = torch.nn.functional.conv_transpose2d(dX, dK, stride=(3, 4))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tK).eval(),
                        dK.grad.data.numpy(),
                        feps=1e-5)
        self.ck_fequals(get_grad(te, tX).eval(),
                        dX.grad.data.numpy(),
                        feps=1e-5)

        X = np.random.randn(2, 4, 5, 5).astype(np.float32)
        K = np.random.randn(4, 3, 5, 8).astype(np.float32)

        tX = val(X)
        tK = val(K)
        tY = rd.op_conv2d_transpose(tX, tK, 3, 4, 6, 8)
        tYf = rd.build_reshape(tY, ((-1, )))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dK = torch.tensor(K, requires_grad=True)
        dY = torch.nn.functional.conv_transpose2d(dX,
                                                  dK,
                                                  stride=(3, 4),
                                                  padding=(6, 8))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tK).eval(),
                        dK.grad.data.numpy(),
                        feps=1e-5)
        self.ck_fequals(get_grad(te, tX).eval(),
                        dX.grad.data.numpy(),
                        feps=1e-5)

        X = np.random.randn(2, 4, 9, 11).astype(np.float32)
        K = np.random.randn(4, 3, 6, 8).astype(np.float32)

        tX = val(X)
        tK = val(K)
        tY = rd.op_conv2d_transpose(tX, tK, 3, 4, 7, 11)
        tYf = rd.build_reshape(tY, ((-1, )))
        te = rd.build_dot_vv(tYf, tYf)

        dX = torch.tensor(X, requires_grad=True)
        dK = torch.tensor(K, requires_grad=True)
        dY = torch.nn.functional.conv_transpose2d(dX,
                                                  dK,
                                                  stride=(3, 4),
                                                  padding=(7, 11))
        dYf = dY.view(-1)
        de = torch.dot(dYf, dYf)
        de.backward()

        self.ck_fequals(tY.eval(), dY.data.numpy())
        self.ck_fequals(get_grad(te, tK).eval(),
                        dK.grad.data.numpy(),
                        feps=1e-5)
        self.ck_fequals(get_grad(te, tX).eval(),
                        dX.grad.data.numpy(),
                        feps=1e-5)