def testNN2(runs=1,
width=3,
data="iris.txt",
iters=20000,
std=True,
trainPct=0.666):
if data == 'gauss':
X, y = multimodalData(numModes=4, numPerMode=30)
# X, y = sklearn.datasets.make_classification()
XY = np.asarray(np.hstack([X, y.reshape((X.shape[0], 1))]))
else:
XY = data_io.read(data)
nclass = len(set(XY[:, -1])) # number of classes
# y has nclass classes (0, ..., nclass-1)
def unary(yi):
return [(1 if i == yi else 0) for i in range(nclass)]
# build a network
u = width
nn = modules.Sequential([
modules.Linear(XY.shape[1] - 1, u),
modules.Tanh(),
modules.Linear(u, u),
modules.Tanh(),
modules.Linear(u, nclass),
modules.SoftMax()
])
results = {False: [], True: []}
for run in range(runs):
Xtrain, ytrain, Xtest, ytest = splitByClass(XY, trainPct)
# Map into n softmax outputs
Ytrain = np.array([unary(yi) for yi in ytrain])
for rms in (False, True):
# train the network.
nn.clean()
nn.train2(np.asarray(Xtrain),
np.asarray(Ytrain),
batchsize=1,
iters=iters,
lrate_decay_step=1000,
rms=rms,
momentum=(0.9 if rms else None))
errors = predictionErrors(nn, Xtest, ytest)
accuracy = 1.0 - (float(errors) / Xtest.shape[0])
print 'RMS', rms, 'Prediction accuracy', accuracy
results[rms].append(accuracy)
print 'Results', results
print 'Average accuracy', 'rms=False', sum(
results[False]) / runs, 'rms=True', sum(results[True]) / runs,
def test_forwardWrongInputListDimension(self):
m = M.Tanh()
try:
m.forward(FloatTensor([[2]]), FloatTensor([[2]]))
except ValueError:
return 0
return 1
def __init__(self):
super(SimpleCNN, self).__init__()
self.conv1 = nn.Conv2d(in_channels=3,
out_channels=6,
kernel_size=3,
stride=3,
padding=2)
self.tanh1 = nn.Tanh()
self.conv2 = nn.Conv2d(in_channels=6,
out_channels=10,
kernel_size=3,
stride=3,
padding=3)
self.tanh2 = nn.Tanh()
self.dropout2d = nn.Dropout2d(rate=0.5)
self.flatten = nn.Flatten()
self.linear = nn.Linear(in_dimension=360, out_dimension=10)
self.softmax = nn.Softmax()
self.set_forward()
def test_backwardBeforeForward(self):
input = FloatTensor([[1, -3, 1], [0, -1, 2]])
m = M.Tanh()
try:
output = m.backward(input)
except:
return 0
return 1
def test_forwardCorrectOutput(self):
input = FloatTensor([[1, -3, 1], [0, -1, 2]])
expected_output = FloatTensor([[0.761594155955765,-0.995054753686730,0.761594155955765],\
[0,-0.761594155955765,0.964027580075817]])
m = M.Tanh()
output = m.forward(input)
if not areEqual(output, expected_output, tol=1e-5):
return 1
return 0
def test_backwardCorrectOutput(self):
input = FloatTensor([[1, -3, 1], [0, -1, 2]])
tanh_value = FloatTensor([[0.761594155955765,-0.995054753686730,0.761594155955765],\
[0,-0.761594155955765,0.964027580075817]])
dsigma = 1 - tanh_value * tanh_value
m = M.Tanh()
m.forward(input)
grad = FloatTensor([[1, 2, 3], [3, 2, 1]])
output = m.backward(grad)
output_expected = dsigma * grad
if not areEqual(output, output_expected):
return 1
return 0
import tools.model_io
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 10000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(settings.nrOfPixels, 4),
modules.Tanh(),
modules.Linear(4, 4),
modules.Tanh(),
modules.Linear(4, 2),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
def testNN(iters=10000, width=3, learning_rates=[0.005], std=True):
# Test cases
# X, y = xor()
# X, y = xor_more()
X, y = multimodalData(numModes=4)
# X, y = sklearn.datasets.make_moons(noise=0.25)
# X, y = sklearn.datasets.make_classification()
# y has 2 classes (0, 1)
# Map into 2 softmax outputs
nclass = len(set(listify(y)))
def unary(yi):
return [(1 if i == yi else 0) for i in range(nclass)]
Y = np.array([unary(yi) for yi in y])
#build a network
nd = X.shape[1] # number of features
u = width
nn = modules.Sequential([
modules.Linear(nd, u),
modules.Tanh(),
modules.Linear(u, u),
modules.Tanh(),
modules.Linear(u, nclass),
modules.SoftMax()
])
# train the network.
errors = []
for lrate in learning_rates:
nn.clean()
# Default does not do learning rate decay or early stopping.
#print ('X,Y,',X,Y)
nn.train2(np.asarray(X),
np.asarray(Y),
batchsize=1,
iters=iters,
lrate=lrate,
lrate_decay_step=100000)
errors.append((lrate, predictionErrors(nn, X, y)))
print 'Errors for learning rates', errors
# Plot the last result
if nd > 2: return # only plot two-feature cases
eps = .1
xmin = np.min(X[:, 0]) - eps
xmax = np.max(X[:, 0]) + eps
ymin = np.min(X[:, 1]) - eps
ymax = np.max(X[:, 1]) + eps
ax = tidyPlot(xmin, xmax, ymin, ymax, xlabel='x', ylabel='y')
def fizz(x1, x2):
y = nn.forward(np.array([x1, x2]))
return y[0, 1] # class 1
res = 30 # resolution of plot
ima = np.array([[fizz(xi, yi) for xi in np.linspace(xmin, xmax, res)] \
for yi in np.linspace(ymin, ymax, res)])
im = ax.imshow(np.flipud(ima),
interpolation='none',
extent=[xmin, xmax, ymin, ymax],
cmap='viridis' if std else 'jet')
plt.colorbar(im)
if std:
colors = [('r' if l == 0 else 'g') for l in y]
ax.scatter(X[:, 0],
X[:, 1],
c=colors,
marker='o',
s=80,
edgecolors='none')
else:
pinds = np.where(y == 0)
ninds = np.where(y == 1)
plt.plot(X[pinds[0], 0], X[pinds[0], 1], 'ob')
plt.plot(X[ninds[0], 0], X[ninds[0], 1], 'or')
#Data normalization
mean, std = inputs.mean(), inputs.std()
train_data.sub_(mean).div_(std)
validation_data.sub_(mean).div_(std)
test_data.sub_(mean).div_(std)
#Instantiate the model
Input_Units = 2
Output_Units = 2
Hidden_Units = 25
model = m.Sequential(m.Linear(Input_Units, Hidden_Units), m.ReLU(),
m.Linear(Hidden_Units, Hidden_Units), m.ReLU(),
m.Linear(Hidden_Units, Hidden_Units), m.Tanh(),
m.Linear(Hidden_Units, Output_Units), m.Tanh())
#Instantiate the optimizer
lr = 0.00095
sgd = m.SGD(params=model.param(), lr=lr)
#Train the model
EPOCHS = 150
model, train_error, validation_error = h.train_model(train_data, train_targets,\
validation_data, validation_targets, model, sgd, nb_epochs = EPOCHS)
'''
#Plot both train and validation errors wrt the number of epochs
fig = plt.figure(figsize=(9, 5))
train_xor = True
train_mnist = True
if train_xor:
D,N = 2,200000
#this is the XOR problem.
X = np.random.rand(N,D) #we want [NxD] data
X = (X > 0.5)*1.0
Y = X[:,0] == X[:,1]
Y = (np.vstack((Y, np.invert(Y)))*1.0).T # and [NxC] labels
X += np.random.randn(N,D)*0.1 # add some noise to the data.
#build a network
nn = modules.Sequential([modules.Linear(2,3), modules.Tanh(),modules.Linear(3,15), modules.Tanh(), modules.Linear(15,15), modules.Tanh(), modules.Linear(15,3), modules.Tanh() ,modules.Linear(3,2), modules.SoftMax()])
#train the network.
nn.train(X,Y, batchsize = 5, iters=1000)
acc = np.mean(np.argmax(nn.forward(X), axis=1) == np.argmax(Y, axis=1))
if not np == numpy: # np=cupy
acc = np.asnumpy(acc)
print('model train accuracy is: {:0.4f}'.format(acc))
#save the network
model_io.write(nn, '../xor_net_small_1000.txt')
if train_mnist:
Xtrain = data_io.read('../data/MNIST/train_images.npy')
Ytrain = data_io.read('../data/MNIST/train_labels.npy')
Xtest = data_io.read('../data/MNIST/test_images.npy')
loadData.normalize_data(test_input)
########## modules and model #########
# define optimizers and losses as list, to be able to juggle with them
optimizers = [optim.SGDOptimizer, optim.SGDmomOptimizer, optim.AdamOptimizer, optim.BFGSOptimizer]
losses = [modules.LossMSE]
# define layers and activations
Lin1 = modules.Linear(2,25)
Lin2 = modules.Linear(25,25)
Lin3 = modules.Linear(25,2)
act1 = modules.ReLU()
act2 = modules.ReLU()
act3 = modules.Tanh()
#act4 = modules.Sigmoid()
# combine the layers together
layers = [
Lin1,
act1,
Lin2,
act2,
Lin3,
act3]
# set parameters for the run
lr = 0.005 # learning rate, for BFGS multiply by 10 to 100
epochs = 250 # epochs for the run
mini_batch_size = 50 # mini_batch_size for the run
