def test_compare1(self): rate = 0.4 x = np.random.rand(10, 2) t = np.zeros((10)).astype(int) layer = dezero.layers.Linear(2, 3, nobias=True) layer.W.data = np.ones_like(layer.W.data) optimizer = dezero.optimizers.SGD().setup(layer) optimizer.add_hook(dezero.optimizers.WeightDecay(rate=rate)) layer.cleargrads() y = layer(x) y = F.softmax_cross_entropy(y, t) y.backward() optimizer.update() W0 = layer.W.data.copy() layer.W.data = np.ones_like(layer.W.data) optimizer.hooks.clear() layer.cleargrads() y = layer(x) y = F.softmax_cross_entropy(y, t) + rate / 2 * (layer.W**2).sum() y.backward() optimizer.update() W1 = layer.W.data self.assertTrue(array_allclose(W0, W1))
def test_forward1(self): x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32) t = np.array([3, 0]).astype(np.int32) y = F.softmax_cross_entropy(x, t) y2 = CF.softmax_cross_entropy(x, t) res = np.allclose(y.data, y2.data) self.assertTrue(res)
def test_MNIST(self): max_epoch = 5 batch_size = 100 hidden_size = 1000 train_set = dezero.datasets.MNIST(train=True) test_set = dezero.datasets.MNIST(train=False) train_loader = DataLoader(train_set, batch_size) test_loader = DataLoader(test_set, batch_size, shuffle=False) #model = MLP((hidden_size, 10)) model = MLP((hidden_size, hidden_size, 10), activation=F.relu) optimizer = optimizers.SGD().setup(model) if dezero.cuda.gpu_enable: train_loader.to_gpu() model.to_gpu() for epoch in range(max_epoch): sum_loss, sum_acc = 0, 0 for x, t in train_loader: y = model(x) loss = F.softmax_cross_entropy(y, t) acc = F.accuracy(y, t) model.cleargrads() loss.backward() optimizer.update() sum_loss += float(loss.data) * len(t) sum_acc += float(acc.data) * len(t) print('epoch: {}'.format(epoch + 1)) print('train loss: {:.4f}, accuracy: {:.4f}'.format( sum_loss / len(train_set), sum_acc / len(train_set))) sum_loss, sum_acc = 0, 0 with dezero.no_grad(): for x, t in test_loader: y = model(x) loss = F.softmax_cross_entropy(y, t) acc = F.accuracy(y, t) sum_loss += float(loss.data) * len(t) sum_acc += float(acc.data) * len(t) print('test loss: {:.4f}, accuracy: {:.4f}'.format( sum_loss / len(test_set), sum_acc / len(test_set)))
def test_softmax_cross_entropy(self): x = Variable(np.array([[1.0, 1.0, 1.0], [3.0, 2.0, 1.0]])) t = Variable(np.array([1, 0])) y = softmax_cross_entropy(x, t) p_1 = np.log(np.exp(1.0) / (3 * np.exp(1.0))) p_2 = np.log(np.exp(3.0) / (np.exp(3.0) + np.exp(2.0) + np.exp(1.0))) expected_ans = -(p_1 + p_2) / 2 assert_array_equal(y.data, np.array(expected_ans))
def test_SoftmaxCrossEntorpy(self): max_epoch = 0 batch_size = 30 hidden_size = 10 lr = 1.0 train_set = Spiral(train=True) test_set = Spiral(train=False) train_loader = DataLoader(train_set, batch_size) test_loader = DataLoader(test_set, batch_size, shuffle=False) model = MLP((hidden_size, hidden_size, hidden_size, 3)) optimizer = optimizers.SGD(lr).setup(model) for epoch in range(max_epoch): sum_loss, sum_acc = 0, 0 for x, t in train_loader: y = model(x) loss = F.softmax_cross_entropy(y, t) acc = F.accuracy(y, t) model.cleargrads() loss.backward() optimizer.update() sum_loss += float(loss.data) * len(t) sum_acc += float(acc.data) * len(t) print('epoch: {}'.format(epoch + 1)) print('train loss: {:.4f}, accuracy: {:.4f}'.format( sum_loss / len(train_set), sum_acc / len(train_set))) sum_loss, sum_acc = 0, 0 with dezero.no_grad(): for x, t in test_loader: y = model(x) loss = F.softmax_cross_entropy(y, t) acc = F.accuracy(y, t) sum_loss += float(loss.data) * len(t) sum_acc += float(acc.data) * len(t) print('test loss: {:.4f}, accuracy: {:.4f}'.format( sum_loss / len(test_set), sum_acc / len(test_set)))
def test_forward(self): x = Variable( np.array([[0.2, -0.4], [0.3, 0.5], [1.3, -3.2], [2.1, 0.3]])) t = np.array([1, 0, 1, 0]) y = F.softmax_cross_entropy(x, t) y_expected = np.exp(x.data) / \ np.exp(x.data).sum(axis=1, keepdims=True) y_expected = -np.log(y_expected) y_expected = y_expected[np.arange(4), t].sum() / 4 assert_almost_equal(y.data, y_expected, decimal=5)
def test_backward(self): x = Variable( np.array([[0.2, -0.4], [0.3, 0.5], [1., -3.2], [2.1, 0.3]])) t = np.array([1, 0, 1, 0]) t_onehot = np.eye(2)[t] L = F.softmax_cross_entropy(x, t) L.backward() m = len(x.data) y = np.exp(x.data) / np.exp(x.data).sum(axis=1, keepdims=True) gx_expected = 1 / m * (y - t_onehot) assert_almost_equal(x.grad.data, gx_expected)
optimizer = optimizers.SGD(lr).setup(model) data_size = len(x) max_iter = math.ceil(data_size / batch_size) for epoch in range(max_epoch): index = np.random.permutation(data_size) sum_loss = 0 for i in range(max_iter): batch_index = index[i * batch_size:(i + 1) * batch_size] batch_x = x[batch_index] batch_t = t[batch_index] y = model(batch_x) loss = F.softmax_cross_entropy(y, batch_t) model.clear_grads() loss.backward() optimizer.update() sum_loss += float(loss.data) * len(batch_t) avg_loss = sum_loss / data_size print('epoch %d, loss %.2f' % (epoch + 1, avg_loss)) # Plot boundary area the model predict h = 0.001 x_min, x_max = x[:, 0].min() - .1, x[:, 0].max() + .1 y_min, y_max = x[:, 1].min() - .1, x[:, 1].max() + .1 xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) X = np.c_[xx.ravel(), yy.ravel()]
# /root/.dezero/t10k-images-idx3-ubyte.gz 경로에 있음 train_set = dezero.datasets.MNIST(train=True) test_set = dezero.datasets.MNIST(train=False) train_loader = DataLoader(train_set, batch_size) test_loader = DataLoader(test_set, batch_size, shuffle=False) # model = MLP((hidden_size, 10)) model = MLP((hidden_size, hidden_size, 10), activation=F.relu) optimizer = SGD().setup(model) for epoch in range(max_epoch): sum_loss, sum_acc = 0, 0 for x, t in train_loader: y = model(x) loss = F.softmax_cross_entropy(y, t) acc = F.accuracy(y, t) model.cleargrads() loss.backward() optimizer.update() sum_loss += float(loss.data) * len(t) sum_acc += float(acc.data) * len(t) print("epoch: {}".format(epoch + 1)) print("train loss: {:.4f}, accuracy: {:.4f}".format( sum_loss / len(train_set), sum_acc / len(train_set))) sum_loss, sum_acc = 0, 0 with dezero.no_grad(): # 기울기 불필요 모드 for x, t in test_loader: # 테스트용 미니배치 데이터
def test_backward3(self): N, CLS_NUM = 100, 10 x = np.random.randn(N, CLS_NUM) t = np.random.randint(0, CLS_NUM, (N, )) f = lambda x: F.softmax_cross_entropy(x, t) self.assertTrue(gradient_check(f, x))
def test_backward1(self): x = np.array([[-1, 0, 1, 2], [2, 0, 1, -1]], np.float32) t = np.array([3, 0]).astype(np.int32) f = lambda x: F.softmax_cross_entropy(x, Variable(t)) self.assertTrue(gradient_check(f, x))
def test_dont_divide_by_zero(self): x = np.array([[1., 0., 300000., 400000.]]) t = np.array([1]) L = F.softmax_cross_entropy(x, t) self.assertFalse(np.isnan(L.data) or np.isinf(L.data))
def test_dont_overflow(self): x = Variable(np.array([[1.0, 100.0, 10000.0]])) t = np.array([0]) L = F.softmax_cross_entropy(x, t) self.assertFalse(np.isnan(L.data) or np.isinf(L.data))