class NN(object):
def __init__(self, in_layer_size, out_layer_size):
self.fc1 = Linear(in_layer_size, out_layer_size, bias=False)
self.ac1 = Tanh()
def forward(self, x):
s1 = self.fc1.forward(x)
a1 = self.ac1.forward(s1)
return a1
def update(self, params):
#print("W:", params[0].shape)
self.fc1.update([params[0]])
if len(params) > 1:
#print("R:", len([params[1]]))
self.ac1.update(params[1])
#print("W:",self.fc1.param()[0][0])
#print("dW:",self.fc1.param()[0][1])
def backward(self, dL_dy):
'''
output dy/dw2 = d(f(wx+b))/dw = x
output dy/dw1 = d(f(wx+b))/dw = x
'''
#print(dL_dy)
dL_ds = self.ac1.backward(dL_dy)
dL_dy0 = self.fc1.backward(dL_ds)
#print(dL_dy0)
return dL_dy0
def param(self):
return [self.fc1.param()[0], self.ac1.param()[0]]
def _read_txt_old(path):
print('loading plain text model from', path)
with open(path, 'rb') as f:
content = f.read().split('\n')
modules = []
c = 0
line = content[c]
while len(line) > 0:
if line.startswith(
Linear.__name__
): # @UndefinedVariable import error suppression for PyDev users
lineparts = line.split()
m = int(lineparts[1])
n = int(lineparts[2])
mod = Linear(m, n)
for i in range(m):
c += 1
mod.W[i, :] = np.array([
float(val) for val in content[c].split()
if len(val) > 0
])
c += 1
mod.B = np.array([float(val) for val in content[c].split()])
modules.append(mod)
elif line.startswith(
Rect.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(Rect())
elif line.startswith(
Tanh.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(Tanh())
elif line.startswith(
SoftMax.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(SoftMax())
elif line.startswith(
BinStep.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(BinStep())
elif line.startswith(
NegAbs.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(NegAbs())
else:
raise ValueError('Layer type ' +
[s for s in line.split() if len(s) > 0][0] +
' not supported by legacy plain text format.')
c += 1
line = content[c]
return Sequential(modules)
def _read_txt_helper(path):
with open(path, 'rb') as f:
content = f.read().split('\n')
modules = []
c = 0
line = content[c]
while len(line) > 0:
if line.startswith(
Linear.__name__
): # @UndefinedVariable import error suppression for PyDev users
'''
Format of linear layer
Linear <rows_of_W> <columns_of_W>
<flattened weight matrix W>
<flattened bias vector>
'''
_, m, n = line.split()
m = int(m)
n = int(n)
layer = Linear(m, n)
layer.W = np.array([
float(weightstring)
for weightstring in content[c + 1].split()
if len(weightstring) > 0
]).reshape((m, n))
layer.B = np.array([
float(weightstring)
for weightstring in content[c + 2].split()
if len(weightstring) > 0
])
modules.append(layer)
c += 3 # the description of a linear layer spans three lines
elif line.startswith(
Convolution.__name__
): # @UndefinedVariable import error suppression for PyDev users
'''
Format of convolution layer
Convolution <rows_of_W> <columns_of_W> <depth_of_W> <number_of_filters_W> <stride_axis_0> <stride_axis_1>
<flattened filter block W>
<flattened bias vector>
'''
_, h, w, d, n, s0, s1 = line.split()
h = int(h)
w = int(w)
d = int(d)
n = int(n)
s0 = int(s0)
s1 = int(s1)
layer = Convolution(filtersize=(h, w, d, n),
stride=(s0, s1))
layer.W = np.array([
float(weightstring)
for weightstring in content[c + 1].split()
if len(weightstring) > 0
]).reshape((h, w, d, n))
layer.B = np.array([
float(weightstring)
for weightstring in content[c + 2].split()
if len(weightstring) > 0
])
modules.append(layer)
c += 3 #the description of a convolution layer spans three lines
elif line.startswith(
SumPool.__name__
): # @UndefinedVariable import error suppression for PyDev users
'''
Format of sum pooling layer
SumPool <mask_heigth> <mask_width> <stride_axis_0> <stride_axis_1>
'''
_, h, w, s0, s1 = line.split()
h = int(h)
w = int(w)
s0 = int(s0)
s1 = int(s1)
layer = SumPool(pool=(h, w), stride=(s0, s1))
modules.append(layer)
c += 1 # one line of parameterized layer description
elif line.startswith(
MaxPool.__name__
): # @UndefinedVariable import error suppression for PyDev users
'''
Format of max pooling layer
MaxPool <mask_heigth> <mask_width> <stride_axis_0> <stride_axis_1>
'''
_, h, w, s0, s1 = line.split()
h = int(h)
w = int(w)
s0 = int(s0)
s1 = int(s1)
layer = MaxPool(pool=(h, w), stride=(s0, s1))
modules.append(layer)
c += 1 # one line of parameterized layer description
elif line.startswith(
Flatten.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(Flatten())
c += 1 #one line of parameterless layer description
elif line.startswith(
Rect.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(Rect())
c += 1 #one line of parameterless layer description
elif line.startswith(
Tanh.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(Tanh())
c += 1 #one line of parameterless layer description
elif line.startswith(
SoftMax.__name__
): # @UndefinedVariable import error suppression for PyDev users
modules.append(SoftMax())
c += 1 #one line of parameterless layer description
else:
raise ValueError(
'Layer type identifier' +
[s for s in line.split() if len(s) > 0][0] +
' not supported for reading from plain text file')
#skip info of previous layers, read in next layer header
line = content[c]
return Sequential(modules)
train_input, train_target = generate_disc_set(1000)
test_input, test_target = generate_disc_set(1000)
batch_size = 100
num_batches = len(train_input) // batch_size
# Reset the seeds before each model creation so that
# parameters are initialized the same for a fair comparison.
torch.manual_seed(0)
relu = Sequential(Linear(2, 25), ReLU(), Linear(25, 25), ReLU(),
Linear(25, 25), ReLU(), Linear(25, 2))
torch.manual_seed(0)
tanh = Sequential(Linear(2, 25), Tanh(), Linear(25, 25), Tanh(),
Linear(25, 25), Tanh(), Linear(25, 2))
criterion = MSE()
def fit(model):
optimizer = SGD(model.parameters(), model.grads(), lr=0.1)
losses = []
print('Epoch | Loss')
for epoch in range(500):
epoch_loss = 0
for b in range(num_batches):
batch_input = train_input[b * batch_size:(b + 1) * batch_size]
batch_target = train_target[b * batch_size:(b + 1) * batch_size]
"""This file declares the models to be used for testing."""
from modules import Sequential, Linear, ReLU, Tanh, Sigmoid
MODEL1 = Sequential("ReLu", Linear(2, 25), ReLU(), Linear(25, 25), ReLU(),
Linear(25, 25), ReLU(), Linear(25, 2), Sigmoid())
MODEL2 = Sequential("Tanh", Linear(2, 25), Tanh(), Linear(25, 25), Tanh(),
Linear(25, 25), Tanh(), Linear(25, 2), Sigmoid())
MODEL3 = Sequential("ReLu + He", Linear(2, 25, "He"), ReLU(),
Linear(25, 25, "He"), ReLU(), Linear(25, 25, "He"), ReLU(),
Linear(25, 2, "He"), Sigmoid())
MODEL4 = Sequential("Tanh + Xavier", Linear(2, 25, "Xavier"), Tanh(),
Linear(25, 25, "Xavier"), Tanh(), Linear(25, 25, "Xavier"),
Tanh(), Linear(25, 2, "Xavier"), Sigmoid())
# Best model is actually almost model 2
MODEL_BEST = Sequential("Best", Linear(2, 25), Tanh(), Linear(25, 25), Tanh(),
Linear(25, 25), Tanh(), Linear(25, 2, "He"), Sigmoid())
def __init__(self, in_layer_size, out_layer_size):
self.fc1 = Linear(in_layer_size, out_layer_size, bias=False)
self.ac1 = Tanh()
nn = NN2(X.shape[1], 30, Y.shape[1])
optim_nn = LossMSE()
trainer = Trainer(nn, optim_nn, v=True)
iterations = 200
eta = 0.0001
# In[16]:
cost_nn = trainer.trainBatchGD(X, Y, iterations, eta=eta)
plotCostAndData(nn, X, Y, cost_nn)
# ### Sequential
# In[18]:
nn_seq = Sequential(Linear(D_in, H1), Tanh(), Linear(H1, H2), Tanh(),
Linear(H2, D_out))
optim_nn = LossMSE()
trainer = Trainer(nn_seq, optim_nn, v=True)
iterations = 300
eta = 0.0008
nn_seq.modules
# In[19]:
cost_nn_seq = trainer.trainBatchGD(X, Y, iterations, eta=eta)
plotCostAndData(nn_seq, X, Y, cost_nn_seq)
# ## Circular input
