def nn(nfeature, nclass, nunits, lamb, seed, learning_rate, beta1, beta2,
epsilon):
np.random.seed(seed)
model = NeuralNetwork.NeuralNetwork(nfeature)
model.add_layer(nunits, "relu", lamb)
model.add_layer(nclass, "softmax", lamb)
optimizer = Optimizer.Adam(learning_rate, beta1, beta2, epsilon)
model.compile("crossentropy", optimizer)
return model
def train_comparison(args, theta, dataset):
hyper_params = {k: args[k] for k in ['b_size']}
hyper_params['to_fix'] = [
] # a selection of parameters can be fixed, e.g. the word embeddings
# initialize optimizer with learning rate (other hyperparams: default values)
opt = args['optimizer']
if opt == 'adagrad':
optimizer = Optimizer.Adagrad(theta,
lr=args['learningrate'],
lambda_l2=args['lambda'])
elif opt == 'adam':
optimizer = Optimizer.Adam(theta,
lr=args['learningrate'],
lambda_l2=args['lambda'])
elif opt == 'sgd':
optimizer = Optimizer.SGD(theta,
lr=args['learningrate'],
lambda_l2=args['lambda'])
else:
raise RuntimeError("No valid optimizer chosen")
# train model
evals = plain_train(optimizer,
dataset,
hyper_params,
n_epochs=args['n_epochs'],
outdir=args['out_dir'])
# store learned model
store_theta(
theta, os.path.join(args['out_dir'], 'comparisonFinalModel.theta.pik'))
# run final evaluation
for name, tb in dataset.iteritems():
print('Evaluation on ' + name + ' data (' + str(len(tb.examples)) +
' examples)')
tb.evaluate(optimizer.theta, verbose=1)
# create convergence plot
for name, eval in evals.items():
toplot = [e[key] for e in eval for key in e if 'loss' in key]
plt.plot(xrange(len(toplot)), toplot, label=name)
plt.legend()
plt.title([key for key in eval[0].keys() if 'loss' in key][0])
plt.savefig(os.path.join(args['out_dir'], 'comparisonConvergence.png'))
# Change random seed to get different random numbers: seed (integer)
# Change number of training data samples: ntrain up to 60000
# Change number of validation data samples: nvalid up to 10000
# Change learning rate for optimization: learning_rate >0
# Change number of iterations: niterations
seed = 10
ntrain = 6000
nvalid = 1000
learning_rate = 0.02
niteration = 40
# (1) Set up data
nclass = 10
Xtrain, Ytrain, Xvalid, Yvalid = load_mnist.load_mnist(ntrain, nvalid)
# (2) Define model
nfeature = Xtrain.shape[0]
np.random.seed(seed)
model = NeuralNetwork.NeuralNetwork(nfeature)
model.add_layer(128, "relu")
model.add_layer(nclass, "softmax")
# (3) Compile model
optimizer = Optimizer.Adam(learning_rate, 0.9, 0.999, 1e-7)
model.compile("crossentropy", optimizer)
model.summary()
# (4) Train model
history = model.fit(Xtrain, Ytrain, niteration)
# (5) Predictions and plotting
# plot data, loss, and animation of results
Yvalid_pred = model.predict(Xvalid)
plot_results.plot_data_mnist(Xtrain, Ytrain)
plot_results.plot_results_history(history, ["loss"])
plot_results.plot_results_mnist_animation(Xvalid, Yvalid, Yvalid_pred, 25)
import time
# (1) Set up data
ntrain = 6000
nvalid = 1000
nclass = 10
Xtrain,Ytrain,Xvalid,Yvalid = load_mnist.load_mnist(ntrain,nvalid)
# (2) Define model
nfeature = Xtrain.shape[0]
np.random.seed(10)
lamb = 0.0
model = NeuralNetwork.NeuralNetwork(nfeature)
model.add_layer(128,"relu",lamb)
model.add_layer(nclass,"softmax",lamb)
# (3) Compile model
optimizer = Optimizer.Adam(0.02,0.9,0.999,1e-7)
model.compile("crossentropy",optimizer)
model.summary()
# (4) Train model
epochs = 40
time_start = time.time()
history = model.fit(Xtrain,Ytrain,epochs,batch_size=ntrain,validation_data=(Xvalid,Yvalid))
time_end = time.time()
print("Train time: {}".format(time_end - time_start))
# (5) Predictions and plotting
# confusion matrix
print("Metrics for Validation Dataset")
Yvalid_pred = model.predict(Xvalid)
metrics.confusion_matrix(Yvalid,Yvalid_pred,nclass)
# plot loss, accuracy, and animation of results
plot_results.plot_results_history(history,["loss","valid_loss"])
y_test = one_hot[50000:60000]
x_test = x_train[50000:60000]
x_train = x_train[:50000]
# initialize parameters
num_classes = y_train.shape[1]
hidden = 50
x_train = x_train.reshape(x_train.shape[0], 1, 28, 28)
x_test = x_test.reshape(x_test.shape[0], 1, 28, 28)
print(x_train.shape)
model = Model()
model.addlayer(ConvLayer(32, (3, 3)), name='conv1')
model.addlayer(Layers.ReLU(), name='relu1')
model.addlayer(Layers.Dropout(), name='dropout1')
model.addlayer(Layers.Flatten(), name='flatten')
model.addlayer(Layers.MulLayer(10), name="w1")
model.addlayer(Layers.AddLayer(10), name='b1')
model.addlayer(Layers.ReLU(), name='relu3')
model.addlayer(Layers.Dropout(0.5), name='dropout3')
model.addlayer(Layers.SoftmaxLayer(), name='softmax')
optimizer = Optimizer.Adam(batch_size=32)
model.train(x_train, y_train, optimizer, 10000, 0.01)
model.save()
print("--TRAIN EVAL--")
model.eval(x_train, y_train)
print("--TEST EVAL--")
model.eval(x_test, y_test)
