def test_dnn(self):
class dnn_mnist_model(nn.Module):
def __init__(self):
super(dnn_mnist_model, self).__init__()
self.fc1 = nn.Linear(28 * 28, 256)
self.fc2 = nn.Linear(256, 10)
self.relu1 = nn.ReLU()
self.relu2 = nn.ReLU()
def forward(self, X):
X = self.fc1(X)
X = self.relu1(X)
X = self.fc2(X)
X = self.relu2(X)
return X
x, y, tx, ty = load_mnist()
x = x.reshape((-1, BATCHSIZE, 28 * 28))
y = y.reshape((-1, BATCHSIZE, 1))
tx = tx.reshape((-1, 1, 28 * 28))
ty = ty.reshape((-1, 1, 1))
model = dnn_mnist_model()
self.assertIsInstance(model, nn.Module)
t_x = madml.tensor(x / 225.)
t_y = madml.tensor(y).onehot(label_count=10)
# loss_fn = nn.MSELoss()
loss_fn = nn.CrossEntropyLoss(with_logit=True)
optim = optimizer.Adam(model.parameters(), lr=1e-3)
train_loop(model, loss_fn, optim, t_x, t_y)
test_x = madml.tensor(tx / 255.)
test_y = madml.tensor(ty)
acc = test_loop(model, test_x, test_y)
print(sum(acc) / len(acc))
def test_identity(self):
class identity_model(nn.Module):
def __init__(self):
super(identity_model, self).__init__()
self.fc1 = nn.Linear(32, 32)
self.fc2 = nn.Linear(32, 32)
def forward(self, X):
X = self.fc1(X)
X = self.fc2(X)
return X
model = identity_model()
self.assertIsInstance(model, nn.Module)
x = np.ones((2, 32))
t_x = madml.tensor(x)
t_y = madml.tensor(x.copy())
loss_fn = nn.MSELoss()
optim = optimizer.Adam(model.parameters(), lr=1e-2)
for i in range(100):
optim.zero_grad()
logit = model(t_x)
loss = loss_fn(logit, t_y)
loss.backward()
optim.step()
print('===', i, logit.shape, loss.host_data, loss_fn.accuracy())
self.assertTrue(loss_fn.accuracy() > 0.9)
def test_cnn(self):
class cnn_mnist_model(nn.Module):
def __init__(self):
super(cnn_mnist_model, self).__init__()
self.conv1 = nn.Conv2d(1, 32, 3, padding=1)
self.pool = nn.MaxPool2d(2, 2)
self.conv2 = nn.Conv2d(32, 48, 3)
self.fc1 = nn.Linear(48 * 12 * 12, 120)
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84, 10)
self.relu1 = nn.ReLU()
self.relu2 = nn.ReLU()
self.relu3 = nn.ReLU()
self.relu4 = nn.ReLU()
def forward(self, X):
X = self.conv1(X)
X = self.relu1(X)
X = self.pool(X) # 32 x 14 x 14
X = self.conv2(X) # 46 x 12 x 12
X = self.relu2(X)
X.flatten()
X = self.fc1(X)
X = self.relu3(X)
X = self.fc2(X)
X = self.relu4(X)
X = self.fc3(X)
return X
x, y, tx, ty = load_mnist()
x = x.reshape((-1, BATCHSIZE, 1, 28, 28))
y = y.reshape((-1, BATCHSIZE, 1))
tx = tx.reshape((-1, 1, 1, 28, 28))
ty = ty.reshape((-1, 1, 1))
model = cnn_mnist_model()
self.assertIsInstance(model, nn.Module)
t_x = madml.tensor(x / 225.)
t_y = madml.tensor(y).onehot(label_count=10)
# loss_fn = nn.MSELoss()
loss_fn = nn.CrossEntropyLoss(with_logit=True)
optim = optimizer.Adam(model.parameters(), lr=1e-3)
train_loop(model, loss_fn, optim, t_x, t_y)
test_x = madml.tensor(tx / 255.)
test_y = madml.tensor(ty)
acc = test_loop(model, test_x, test_y)
print(sum(acc) / len(acc))
def identity_train_loop(model_class=identity_model):
model = model_class()
x = np.ones((2, 32))
t_x = madml.tensor(x)
t_y = madml.tensor(x.copy())
loss_fn = nn.MSELoss()
optim = optimizer.Adam(model.parameters(), lr=1e-2)
logit = None
for _ in range(100):
optim.zero_grad()
logit = model(t_x)
loss = loss_fn(logit, t_y)
loss.backward()
optim.step()
print(logit.shape, loss.host_data, loss_fn.accuracy())
print(logit.host_data)
def test_spiral(self):
from numpy import pi
# import matplotlib.pyplot as plt
N = 400
theta = np.sqrt(np.random.rand(N)) * 2 * pi # np.linspace(0,2*pi,100)
r_a = 2 * theta + pi
data_a = np.array([np.cos(theta) * r_a, np.sin(theta) * r_a]).T
x_a = data_a + np.random.randn(N, 2)
res_a = np.append(x_a, np.zeros((N, 1)), axis=1)
r_b = -2 * theta - pi
data_b = np.array([np.cos(theta) * r_b, np.sin(theta) * r_b]).T
x_b = data_b + np.random.randn(N, 2)
res_b = np.append(x_b, np.ones((N, 1)), axis=1)
res = np.append(res_a, res_b, axis=0)
np.random.shuffle(res)
ax1 = plt.subplot(121)
ax1.margins(0.05)
ax1.scatter(x_a[:, 0], x_a[:, 1])
ax1.scatter(x_b[:, 0], x_b[:, 1])
# np.savetxt("result.csv", res[0], delimiter=",", header="x,y,label", comments="", fmt='%.5f')
class spiral_model(nn.Module):
def __init__(self):
super(spiral_model, self).__init__()
self.fc1 = nn.Linear(2, 16)
self.fc2 = nn.Linear(16, 16)
self.fc3 = nn.Linear(16, 2)
self.tanh1 = nn.tanh()
self.tanh2 = nn.tanh()
self.sig = nn.ReLU()
def forward(self, X):
X = self.fc1(X)
X = self.tanh1(X)
X = self.fc2(X)
X = self.tanh2(X)
X = self.fc3(X)
X = self.sig(X)
return X
model = spiral_model()
self.assertIsInstance(model, nn.Module)
x = res[..., :-1]
y = res[..., 2]
t_x = madml.tensor(x)
t_y = madml.tensor(y)
t_y = t_y.onehot(2)
t_y.reshape([800, 2])
loss_fn = nn.CrossEntropyLoss(with_logit=True)
# loss_fn = nn.MSELoss()
optim = optimizer.Adam(model.parameters(), lr=1e-2)
logits = None
for i in range(100):
optim.zero_grad()
logit = model(t_x)
logits = logit.host_data
loss = loss_fn(logit, t_y)
loss.backward()
optim.step()
print('===', i, logit.shape, loss.host_data)
logits = np.argmax(logits, axis=-1)
result = res[:, :-1]
ax2 = plt.subplot(122)
ax2.scatter(result[logits == 0.][:, 0], result[logits == 0.][:, 1])
ax2.scatter(result[logits == 1.][:, 0], result[logits == 1.][:, 1])
# plt.savefig('input_output.png')
acc = (logits - y).mean()
print(1. - acc)
self.assertTrue(1.0 - acc > 0.9)