def testNN2(runs=1,
width=3,
data="iris.txt",
iters=20000,
std=True,
trainPct=0.666):
if data == 'gauss':
X, y = multimodalData(numModes=4, numPerMode=30)
# X, y = sklearn.datasets.make_classification()
XY = np.asarray(np.hstack([X, y.reshape((X.shape[0], 1))]))
else:
XY = data_io.read(data)
nclass = len(set(XY[:, -1])) # number of classes
# y has nclass classes (0, ..., nclass-1)
def unary(yi):
return [(1 if i == yi else 0) for i in range(nclass)]
# build a network
u = width
nn = modules.Sequential([
modules.Linear(XY.shape[1] - 1, u),
modules.Tanh(),
modules.Linear(u, u),
modules.Tanh(),
modules.Linear(u, nclass),
modules.SoftMax()
])
results = {False: [], True: []}
for run in range(runs):
Xtrain, ytrain, Xtest, ytest = splitByClass(XY, trainPct)
# Map into n softmax outputs
Ytrain = np.array([unary(yi) for yi in ytrain])
for rms in (False, True):
# train the network.
nn.clean()
nn.train2(np.asarray(Xtrain),
np.asarray(Ytrain),
batchsize=1,
iters=iters,
lrate_decay_step=1000,
rms=rms,
momentum=(0.9 if rms else None))
errors = predictionErrors(nn, Xtest, ytest)
accuracy = 1.0 - (float(errors) / Xtest.shape[0])
print 'RMS', rms, 'Prediction accuracy', accuracy
results[rms].append(accuracy)
print 'Results', results
print 'Average accuracy', 'rms=False', sum(
results[False]) / runs, 'rms=True', sum(results[True]) / runs,
def testNN(iters=10000, width=3, learning_rates=[0.005], std=True):
# Test cases
# X, y = xor()
# X, y = xor_more()
X, y = multimodalData(numModes=4)
# X, y = sklearn.datasets.make_moons(noise=0.25)
# X, y = sklearn.datasets.make_classification()
# y has 2 classes (0, 1)
# Map into 2 softmax outputs
nclass = len(set(listify(y)))
def unary(yi):
return [(1 if i == yi else 0) for i in range(nclass)]
Y = np.array([unary(yi) for yi in y])
#build a network
nd = X.shape[1] # number of features
u = width
nn = modules.Sequential([
modules.Linear(nd, u),
modules.Tanh(),
modules.Linear(u, u),
modules.Tanh(),
modules.Linear(u, nclass),
modules.SoftMax()
])
# train the network.
errors = []
for lrate in learning_rates:
nn.clean()
# Default does not do learning rate decay or early stopping.
#print ('X,Y,',X,Y)
nn.train2(np.asarray(X),
np.asarray(Y),
batchsize=1,
iters=iters,
lrate=lrate,
lrate_decay_step=100000)
errors.append((lrate, predictionErrors(nn, X, y)))
print 'Errors for learning rates', errors
# Plot the last result
if nd > 2: return # only plot two-feature cases
eps = .1
xmin = np.min(X[:, 0]) - eps
xmax = np.max(X[:, 0]) + eps
ymin = np.min(X[:, 1]) - eps
ymax = np.max(X[:, 1]) + eps
ax = tidyPlot(xmin, xmax, ymin, ymax, xlabel='x', ylabel='y')
def fizz(x1, x2):
y = nn.forward(np.array([x1, x2]))
return y[0, 1] # class 1
res = 30 # resolution of plot
ima = np.array([[fizz(xi, yi) for xi in np.linspace(xmin, xmax, res)] \
for yi in np.linspace(ymin, ymax, res)])
im = ax.imshow(np.flipud(ima),
interpolation='none',
extent=[xmin, xmax, ymin, ymax],
cmap='viridis' if std else 'jet')
plt.colorbar(im)
if std:
colors = [('r' if l == 0 else 'g') for l in y]
ax.scatter(X[:, 0],
X[:, 1],
c=colors,
marker='o',
s=80,
edgecolors='none')
else:
pinds = np.where(y == 0)
ninds = np.where(y == 1)
plt.plot(X[pinds[0], 0], X[pinds[0], 1], 'ob')
plt.plot(X[ninds[0], 0], X[ninds[0], 1], 'or')
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 10000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(32**2, 1),
modules.NegAbs(),
modules.Tanh(),
modules.Linear(1, 2),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
import tools.model_io
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 1000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(settings.nrOfPixels, 4),
# modules.Rect(),
# modules.Linear(4, 4),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
def roar_kar(keep, random=False, train_only=False):
logdir = 'tf_logs/standard/'
def get_savedir():
savedir = logdir.replace('tf_logs', 'KAR' if keep else 'ROAR')
if not os.path.exists(savedir):
os.makedirs(savedir)
return savedir
# ratio = 0.1
percentiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
attribution_methods = ['normal', 'LRP', 'proposed_method']
if not train_only:
DNN = model_io.read('../models/MNIST/LeNet-5.nn')
for v in attribution_methods:
batch_size = 128
print("{} Step is start".format(v))
if random:
print("{} percentile Remove".format(v))
occlude_dataset(DNN=DNN,
attribution=v,
percentiles=percentiles,
random=True,
keep=keep,
batch_size=batch_size,
savedir=get_savedir())
else:
print("{} Random Remove".format(v))
occlude_dataset(DNN=DNN,
attribution=v,
percentiles=percentiles,
random=False,
keep=keep,
batch_size=batch_size,
savedir=get_savedir())
print("{} : occlude step is done".format(v))
print("ress record")
ress = {k: [] for k in attribution_methods}
for _ in range(3):
for v in attribution_methods:
res = []
for p in percentiles:
occdir = get_savedir() + '{}_{}_{}.pickle'.format('{}', v, p)
occdir_y = get_savedir() + '{}_{}_{}_{}.pickle'.format(
'{}', v, p, 'label')
data_train = unpickle(occdir.format('train'))
# data_test = unpickle(occdir.format('test'))
Xtrain = np.array(data_train)
Ytrain = unpickle(occdir_y.format('train'))
Ytrain = np.array(Ytrain)
Xtest = data_io.read('../data/MNIST/test_images.npy')
Ytest = data_io.read('../data/MNIST/test_labels.npy')
print("check : {}".format(Ytrain.shape))
Xtest = scale(Xtest)
Xtest = np.reshape(Xtest, [Xtest.shape[0], 28, 28, 1])
Xtest = np.pad(Xtest, ((0, 0), (2, 2), (2, 2), (0, 0)),
'constant',
constant_values=(-1., ))
Ix = Ytest[:, 0].astype(int)
Ytest = np.zeros([Xtest.shape[0], np.unique(Ytest).size])
Ytest[np.arange(Ytest.shape[0]), Ix] = 1
print(occdir)
# DNN = model_io.read('../models/MNIST/LeNet-5.nn')
DNN = modules.Sequential([
modules.Convolution(filtersize=(5,5,1,10),stride = (1,1)),\
modules.Rect(),\
modules.SumPool(pool=(2,2),stride=(2,2)),\
modules.Convolution(filtersize=(5,5,10,25),stride = (1,1)),\
modules.Rect(),\
modules.SumPool(pool=(2,2),stride=(2,2)),\
modules.Convolution(filtersize=(4,4,25,100),stride = (1,1)),\
modules.Rect(),\
modules.SumPool(pool=(2,2),stride=(2,2)),\
modules.Convolution(filtersize=(1,1,100,10),stride = (1,1)),\
modules.Flatten()
])
print("training...")
DNN.train(X=Xtrain,\
Y=Ytrain,\
Xval=Xtest,\
Yval=Ytest,\
iters=10**5,\
lrate=0.0001,\
# status = 2,\
batchsize = 128
)
# ypred = DNN.forward(Xtest)
acc = np.mean(
np.argmax(DNN.forward(Xtest), axis=1) == np.argmax(Ytest,
axis=1))
del DNN
print('metric model test accuracy is: {:0.4f}'.format(acc))
res.append(acc)
print("End of {}:training, accuracy...".format(_))
ress[v].append(res)
print("metric...")
res_mean = {v: np.mean(v, axis=0) for v in ress.item()}
print(res_mean)
return res_mean
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 10000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(settings.nrOfPixels, 16),
modules.Tanh(),
modules.Linear(16, 16),
modules.Tanh(),
modules.Linear(16, 3),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
import tools.model_io
import modules
import shutil
import settings
# user init
batchsize = 25
numbIters = 10000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(32**2, 1),
modules.Rect(),
modules.Linear(1, 2),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
import tools.model_io
import modules
import shutil
import settings
# user init
batchsize = 25
numbIters = 20000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([modules.Linear(settings.nrOfPixels, 2),
modules.Rect(),
modules.Linear(2, Y[settings.kinds[0]].shape[-1]),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'
.format(batchsize, numbIters))
train_xor = True
train_mnist = True
if train_xor:
D,N = 2,200000
#this is the XOR problem.
X = np.random.rand(N,D) #we want [NxD] data
X = (X > 0.5)*1.0
Y = X[:,0] == X[:,1]
Y = (np.vstack((Y, np.invert(Y)))*1.0).T # and [NxC] labels
X += np.random.randn(N,D)*0.1 # add some noise to the data.
#build a network
nn = modules.Sequential([modules.Linear(2,3), modules.Tanh(),modules.Linear(3,15), modules.Tanh(), modules.Linear(15,15), modules.Tanh(), modules.Linear(15,3), modules.Tanh() ,modules.Linear(3,2), modules.SoftMax()])
#train the network.
nn.train(X,Y, batchsize = 5, iters=1000)
acc = np.mean(np.argmax(nn.forward(X), axis=1) == np.argmax(Y, axis=1))
if not np == numpy: # np=cupy
acc = np.asnumpy(acc)
print('model train accuracy is: {:0.4f}'.format(acc))
#save the network
model_io.write(nn, '../xor_net_small_1000.txt')
if train_mnist:
Xtrain = data_io.read('../data/MNIST/train_images.npy')
Ytrain = data_io.read('../data/MNIST/train_labels.npy')
Xtest = data_io.read('../data/MNIST/test_images.npy')
#Data normalization
mean, std = inputs.mean(), inputs.std()
train_data.sub_(mean).div_(std)
validation_data.sub_(mean).div_(std)
test_data.sub_(mean).div_(std)
#Instantiate the model
Input_Units = 2
Output_Units = 2
Hidden_Units = 25
model = m.Sequential(m.Linear(Input_Units, Hidden_Units), m.ReLU(),
m.Linear(Hidden_Units, Hidden_Units), m.ReLU(),
m.Linear(Hidden_Units, Hidden_Units), m.Tanh(),
m.Linear(Hidden_Units, Output_Units), m.Tanh())
#Instantiate the optimizer
lr = 0.00095
sgd = m.SGD(params=model.param(), lr=lr)
#Train the model
EPOCHS = 150
model, train_error, validation_error = h.train_model(train_data, train_targets,\
validation_data, validation_targets, model, sgd, nb_epochs = EPOCHS)
'''
#Plot both train and validation errors wrt the number of epochs
fig = plt.figure(figsize=(9, 5))
I = Ytrain[:, 0].astype(int)
Ytrain = np.zeros([Xtrain.shape[0], np.unique(Ytrain).size])
Ytrain[np.arange(Ytrain.shape[0]), I] = 1
I = Ytest[:, 0].astype(int)
Ytest = np.zeros([Xtest.shape[0], np.unique(Ytest).size])
Ytest[np.arange(Ytest.shape[0]), I] = 1
#model a network according to LeNet-5 architecture
lenet = modules.Sequential([
modules.Convolution(filtersize=(5,5,1,10),stride = (1,1)),\
modules.Rect(),\
modules.SumPool(pool=(2,2),stride=(2,2)),\
modules.Convolution(filtersize=(5,5,10,25),stride = (1,1)),\
modules.Rect(),\
modules.SumPool(pool=(2,2),stride=(2,2)),\
modules.Convolution(filtersize=(4,4,25,100),stride = (1,1)),\
modules.Rect(),\
modules.SumPool(pool=(2,2),stride=(2,2)),\
modules.Convolution(filtersize=(1,1,100,10),stride = (1,1)),\
modules.Flatten()
])
#train the network.
lenet.train( X=Xtrain,\
Y=Ytrain,\
Xval=Xtest,\
Yval=Ytest,\
iters=10**6,\
lrate=0.001,\
batchsize=25)
layers = [
Lin1,
act1,
Lin2,
act2,
Lin3,
act3]
# set parameters for the run
lr = 0.005 # learning rate, for BFGS multiply by 10 to 100
epochs = 250 # epochs for the run
mini_batch_size = 50 # mini_batch_size for the run
# initialize loss, model and optimizer
loss = losses[0]() # [0] for MSELoss
model = modules.Sequential(layers, loss)
optimizer = optimizers[2](model, lr) # [2] for Adam optimizer
# train the model
print('\n----------------------------------------')
print(' TRAINING THE MODEL \n\n')
loss_list, train_acc = trainAndTest.train(model, optimizer, loss, train_input, train_target,\
epochs, mini_batch_size, verbose = False)
# print results
print('\n----------------------------------------')
print(' RESULTS \n')
print('learning rate: ', lr, ' || epochs: ', epochs, ' || mini_batch_size: ', mini_batch_size)
print('optimizer: ', optimizer.name)
print('minimum training loss: ',min(loss_list))
print('maximum training accuracy on train: ',max(train_acc)/train_input.size(0)*100,'%')
import tools.model_io
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 100000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(32**2, 2),
modules.Tanh(),
modules.Linear(2, 2),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 10000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(settings.nrOfPixels, 4),
modules.Tanh(),
modules.Linear(4, 4),
modules.Tanh(),
modules.Linear(4, 2),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
train_target = convert_to_one_hot_labels(train_input, train_target)
test_target = convert_to_one_hot_labels(test_input, test_target)
# Avoid vanishing gradient
train_input = 0.9 * train_input
# Normalize data
mean, std = train_input.mean(), train_input.std()
train_input.sub_(mean).div_(std)
test_input.sub_(mean).div_(std)
# Define models
model1 = modules.Sequential(modules.Linear(2, 25),
modules.TanH(),
modules.Linear(25, 25),
modules.TanH(),
modules.Linear(25, 25),
modules.TanH(),
modules.Linear(25, 25),
modules.TanH(),
modules.Linear(25, 2),
modules.MSELoss()
)
model2 = modules.Sequential(modules.Linear(2, 25),
modules.ReLU(),
modules.Linear(25, 25),
modules.ReLU(),
modules.Linear(25, 25),
modules.ReLU(),
modules.Linear(25, 25),
modules.ReLU(),
modules.Linear(25, 2),
modules.MSELoss()
import modules
import shutil
import settings
# user init
batchsize = 10
numbIters = 10000
# load data
X, Y = tools.data_loader.load_data()
# setup neural network
nn = modules.Sequential([
modules.Linear(32**2, 2),
modules.NegAbs(),
modules.BinStep(),
modules.Linear(2, 2),
modules.SoftMax()
])
# train neural network
nn.train(X['train'],
Y['train'],
Xval=X['valid'],
Yval=Y['valid'],
batchsize=batchsize,
iters=numbIters)
# determine training name of neural net
nnName = 'nn_' + nn.name + ('_(batchsize_{}_number_iterations_{})'.format(
batchsize, numbIters))
@version: 1.0 @copyright: Copyright (c) 2015, Sebastian Lapuschkin, Alexander Binder, Gregoire Montavon, Klaus-Robert Mueller @license : BSD-2-Clause ''' import modules import model_io import numpy as np ; na = np.newaxis D,N = 2,200000 #this is the XOR problem. X = np.random.rand(N,D) #we want [NxD] data X = (X > 0.5)*1.0 Y = X[:,0] == X[:,1] Y = (np.vstack((Y, np.invert(Y)))*1.0).T # and [NxC] labels X += np.random.randn(N,D)*0.1 # add some noise to the data. #build a network nn = modules.Sequential([modules.Linear(2,3), modules.Tanh(),modules.Linear(3,15), modules.Tanh(), modules.Linear(15,15), modules.Tanh(), modules.Linear(15,3), modules.Tanh() ,modules.Linear(3,2), modules.SoftMax()]) #train the network. nn.train(X,Y,Xval=X,Yval=Y, batchsize = 5) #save the network #model_io.write(nn, '../xor_net_small_1000.txt')
