def cnn_with_args(eps=0.1,
momentum=0.5,
num_epochs=30,
filter_size=5,
num_filters_1=8,
num_filters_2=16,
batch_size=100):
# Input-output dimensions.
num_channels = 1
num_outputs = 7
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2, num_outputs)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48, 48, 1) * 0.1
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
# Train model.
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps, momentum, num_epochs, batch_size)
# Construct file name
fname = str(eps) + '_' + str(momentum) + '_' + str(num_epochs) + '_' + str(filter_size) + '_' + \
str(num_filters_1) + '_' + str(num_filters_2) + '_' + str(batch_size)
# Save the training statistics.
Save('cnn_model/' + fname, model)
Save('cnn_stats/' + fname, stats)
def main():
"""Trains a CNN."""
model_fname = 'cnn_model.npz'
stats_fname = 'cnn_stats.npz'
# Hyper-parameters. Modify them if needed.
eps = [0.01] #[0.001, 0.01, 0.1, 0.5, 1.0]
momentum = [0.9] #[0.0, 0.45, 0.9]
num_epochs = 30
filter_size = 5
num_filters_1 = 5
num_filters_2 = 2
batch_size = [100] #[500, 1000]
# Input-output dimensions.
num_channels = 1
num_outputs = 7
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48, 48, 1) * 0.1
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
print("passed grad check")
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
dir = str(0.01) + "\\" + str(0.9) + "\\" + str(100) + "\\"
if os.path.exists(dir):
filelist = [f for f in os.listdir(".") if f.endswith(".bak")]
for f in filelist:
os.remove(f)
else:
os.makedirs(dir)
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps[0],
momentum[0], num_epochs, batch_size[0])
# Uncomment if you wish to save the model.
Save(dir + model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(dir + stats_fname, stats)
SavePlot(0, dir + "cross_entropy.png")
SavePlot(1, dir + "accuracy.png")
'''
def main():
"""Trains a NN."""
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [0, 0] #[150,20] [6,2] [200, 140]
eps = [0.01] #[0.001, 0.01, 0.1, 0.5, 1.0]
momentum = [0.0] #[0.0, 0.45, 0.9]
num_epochs = 1000
batch_size = [100] #[1, 10, 100, 500, 1000]
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
print("passed grad check")
for e in eps:
for m in momentum:
for b in batch_size:
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Train model.
dir = str(e) + "\\" + str(m) + "\\" + str(b) + "\\"
if os.path.exists(dir):
filelist = [
f for f in os.listdir(".") if f.endswith(".bak")
]
for f in filelist:
os.remove(f)
else:
os.makedirs(dir)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, e,
m, num_epochs, b)
# Uncomment if you wish to save the model.
Save(dir + model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(dir + stats_fname, stats)
SavePlot(0, dir + "cross_entropy.png")
SavePlot(1, dir + "accuracy.png")
def main():
"""Trains a CNN."""
model_fname = 'cnn_model.npz'
stats_fname = 'cnn_stats.npz'
# Hyper-parameters. Modify them if needed.
eps = 0.01
momentum = 0.9
num_epochs = 30
filter_size = 5
num_filters_1 = 8
num_filters_2 = 16
batch_size = 100
# Input-output dimensions.
num_channels = 1
num_outputs = 7
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48, 48, 1) * 0.1
print('checking W3 and b3')
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
print('checking W2 and b2')
CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
print('checking W1 and b1')
CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
print('done checking')
# Train model.
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps,
momentum, num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save('cnn_model_batch_500_1.npz', model)
# Uncomment if you wish to save the training statistics.
Save('cnn_stats_batch_500_1.npz', stats)
def main():
"""Trains a NN."""
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [16, 32]
eps = 0.007
momentum = 0.0
num_epochs = 1000
batch_size = 100
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
model, stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size)
q35_plot(model, NNForward)
# Uncomment if you wish to save the model.
Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
ShowWeights(model['W1'], 2)
def main():
"""Trains a NN."""
# NPZ is a file format by numpy that provides storage
# of array data using gzip compression.
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Default hyper-parameters.
num_hiddens = [16, 32]
eps = 0.01
momentum = 0.0
num_epochs = 1000
batch_size = 100
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
def save_train():
train_file = '../411a3/train_aug.npz'
X_train, y_train = load_aug_image_train()
X_test, y_test = load_aug_image_validation()
train = {
'X_train': X_train,
'y_train': y_train,
'X_test': X_test,
'y_test': y_test
}
Save(train_file, train)
print('Train data saved to: ' + train_file)
def Optimization():
"""
Try 5 different values of the learning rate from 0.001 to 1.0.
Try 3 values of momentum from 0.0 to 0.9.
Try 5 different mini-batch sizes, from 1 to 1000.
Find the best value of these parameters
"""
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Default hyper-parameters.
num_hiddens = [16, 32]
num_epochs = 1000
eps = 0.01
momentum = 0.0
batch_size = 100
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Try different eps
for eps_i in [0.001, 0.05, 0.1, 0.5, 1.0]:
model = InitNN(num_inputs, num_hiddens, num_outputs)
stats = Train(model, NNForward, NNBackward, NNUpdate, eps_i, momentum,
num_epochs, batch_size)
Save('nn_model_eps' + str(eps_i) + '.npz', model)
Save('nn_stats_eps' + str(eps_i) + '.npz', stats)
# Try different momentum
for momentum_i in [0, 0.5, 0.9]:
model = InitNN(num_inputs, num_hiddens, num_outputs)
stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum_i,
num_epochs, batch_size)
Save('nn_model_momentum' + str(momentum_i) + '.npz', model)
Save('nn_stats_momentum' + str(momentum_i) + '.npz', stats)
# Try different batch size
for batch_size_i in [1, 10, 50, 500, 1000]:
model = InitNN(num_inputs, num_hiddens, num_outputs)
stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size_i)
Save('nn_model_batch_size' + str(batch_size_i) + '.npz', model)
Save('nn_stats_batch_size' + str(batch_size_i) + '.npz', stats)
def main():
"""Trains a CNN."""
'''
configs = {
'default': {
'model_fname': '3.1_cnn_model.npz',
'stats_fname': '3.1_cnn_stats.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - eps=0.001': {
'model_fname': '3.2_cnn_model_eps_0.001.npz',
'stats_fname': '3.2_cnn_stats_eps_0.001.npz',
'eps': 0.001,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - eps=0.01': {
'model_fname': '3.2_cnn_model_eps_0.01.npz',
'stats_fname': '3.2_cnn_stats_eps_0.01.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - eps=0.1': {
'model_fname': '3.2_cnn_model_eps_0.1.npz',
'stats_fname': '3.2_cnn_stats_eps_0.1.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - eps=0.5': {
'model_fname': '3.2_cnn_model_eps_0.5.npz',
'stats_fname': '3.2_cnn_stats_eps_0.5.npz',
'eps': 0.5,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - eps=1.0': {
'model_fname': '3.2_cnn_model_eps_1.0.npz',
'stats_fname': '3.2_cnn_stats_eps_1.0.npz',
'eps': 1.0,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - momentum=0.0': {
'model_fname': '3.2_cnn_model_momentum_0.0.npz',
'stats_fname': '3.2_cnn_stats_momentum_0.0.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - momentum=0.5': {
'model_fname': '3.2_cnn_model_momentum_0.5.npz',
'stats_fname': '3.2_cnn_stats_momentum_0.5.npz',
'eps': 0.1,
'momentum': 0.5,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - momentum=0.9': {
'model_fname': '3.2_cnn_model_momentum_0.9.npz',
'stats_fname': '3.2_cnn_stats_momentum_0.9.npz',
'eps': 0.1,
'momentum': 0.9,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - batch=1': {
'model_fname': '3.2_cnn_model_batch_1.npz',
'stats_fname': '3.2_cnn_stats_batch_1.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 1,
},
'3.2 - batch=10': {
'model_fname': '3.2_cnn_model_batch_10.npz',
'stats_fname': '3.2_cnn_stats_batch_10.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 10,
},
'3.2 - batch=100': {
'model_fname': '3.2_cnn_model_batch_100.npz',
'stats_fname': '3.2_cnn_stats_batch_100.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
},
'3.2 - batch=500': {
'model_fname': '3.2_cnn_model_batch_500.npz',
'stats_fname': '3.2_cnn_stats_batch_500.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 500,
},
'3.2 - batch=1000': {
'model_fname': '3.2_cnn_model_batch_1000.npz',
'stats_fname': '3.2_cnn_stats_batch_1000.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 1000,
},
'3.3 - filter=4,8': {
'model_fname': '3.3_cnn_model_filter_4_8.npz',
'stats_fname': '3.3_cnn_stats_filter_4_8.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 4,
'num_filters_2': 8,
'batch_size': 100,
},
'3.3 - filter=16,32': {
'model_fname': '3.3_cnn_model_filter_16_32.npz',
'stats_fname': '3.3_cnn_stats_filter_16_32.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 16,
'num_filters_2': 32,
'batch_size': 100,
},
'3.3 - filter=16,16': {
'model_fname': '3.3_cnn_model_filter_16_16.npz',
'stats_fname': '3.3_cnn_stats_filter_16_16.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 16,
'num_filters_2': 16,
'batch_size': 100,
},
'origin': {
'model_fname': 'cnn_model.npz',
'stats_fname': 'cnn_stats.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 30,
'filter_size': 5,
'num_filters_1': 8,
'num_filters_2': 16,
'batch_size': 100,
}
}
# Hyper-parameters. Modify them if needed.
# Input-output dimensions.
num_channels = 1
num_outputs = 7
for key, config in configs.items():
# Initialize model.
model = InitCNN(num_channels, config['filter_size'], config['num_filters_1'], config['num_filters_2'],
num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
print(config)
x = np.random.rand(10, 48, 48, 1) * 0.1
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
# Train model.
trained_model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, config['eps'],
config['momentum'], config['num_epochs'], config['batch_size'])
# Uncomment if you wish to save the model.
Save(config['model_fname'], trained_model)
# Uncomment if you wish to save the training statistics.
Save(config['stats_fname'], stats)
'''
model_fname = '3.4_cnn_model_filter_45_45.npz'
stats_fname = '3.4_cnn_stats_filter_45_45.npz'
# Hyper-parameters. Modify them if needed.
eps = 0.1
momentum = 0.0
num_epochs = 30
filter_size = 5
num_filters_1 = 45
num_filters_2 = 45
batch_size = 100
# Input-output dimensions.
num_channels = 1
num_outputs = 7
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48, 48, 1) * 0.1
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
# Train model.
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps,
momentum, num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
def main():
"""Trains a NN."""
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [16, 32] #16
# eps = 0.01
# momentum = 0.0
# num_epochs = 1000
# batch_size = 100
eps = 0.1
momentum = 0.9
num_epochs = 100
batch_size = 100
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname) #load a trained model
inputs_train, inputs_valid, inputs_test, target_train, target_valid, \
target_test = LoadData('toronto_face.npz')
# inputs_train1, inputs_valid1, inputs_test1, target_train1, target_valid1, \
# target_test1 = LoadData('test.npz')
# img = Image.open("testImage.png")
# arr = np.array(img)
# np.savez("test", arr)
# inputs_train1 = arr.T/255
#Uncomment to plot low softmax probabilities (high uncertainity images)
# prediction_stats = NNForward(model, inputs_test)
# prediction = Softmax(prediction_stats['y'])
# plot_uncertain_images(inputs_test, target_test, prediction)
# prediction_stats = NNForward(model, inputs_train1)
# prediction = Softmax(prediction_stats['y'])
# plot_uncertain_images(inputs_train1, target_test, prediction)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save(model_fname, model)
def main():
"""Trains a CNN."""
#model_fname = 'cnn_model.npz'
#stats_fname = 'cnn_stats.npz'
#model_fname = 'cnn_model_32_eps001.npz'
#stats_fname = 'cnn_stats_32_eps001.npz'
#model_fname = 'cnn_model_32_eps01.npz'
#stats_fname = 'cnn_stats_32_eps01.npz'
#model_fname = 'cnn_model_32_eps05.npz'
#stats_fname = 'cnn_stats_32_eps05.npz'
#model_fname = 'cnn_model_32_eps99.npz'
#stats_fname = 'cnn_stats_32_eps99.npz'
#model_fname = 'cnn_model_32_mom09.npz'
#stats_fname = 'cnn_stats_32_mom09.npz'
model_fname = 'cnn_model_34.npz'
stats_fname = 'cnn_stats_34.npz'
# Hyper-parameters. Modify them if needed.
eps = 0.01
momentum = 0.0
num_epochs = 50
#filter_size = 5
#num_filters_1 = 8
#num_filters_2 = 16
filter_size = 5
num_filters_1 = 20
num_filters_2 = 20
batch_size = 100
# Input-output dimensions.
num_channels = 1
num_outputs = 7
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
#x = np.random.rand(10, 48, 48, 1) * 0.1
#CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
#CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
#CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
#CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
#CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
#CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
# Train model.
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps,
momentum, num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
def main():
'''Uncomment to Run on all combinations of hyper parameters and save stats
'''
'''
d_num_hiddens = [16, 32]
num_hiddens = [[5,50],[100,100],[50,5],[5,5],[50,50]]
num_epochs = 1000
num_inputs = 2304
num_outputs = 7
# Check gradient implementation.
print('Checking gradients... 10 times')
for i in range(0,10):
model = InitNN(num_inputs, d_num_hiddens, num_outputs)
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
eps = [.001,.01,.1,.5,1]
d_eps = .01
momentum = [0,.45,.9]
d_mom = 0.0
batch_size=[1,20,300,750,1000]
d_batch_size = 100
for each in eps:
model = InitNN(num_inputs, d_num_hiddens, num_outputs)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, each,
d_mom, num_epochs, d_batch_size)
Save('results/eps_' + str(each) + '_nn.epz',stats)
for each in momentum:
model = InitNN(num_inputs, d_num_hiddens, num_outputs)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, d_eps,
each, num_epochs, d_batch_size)
Save('results/momentum_' + str(each) + '_nn.epz',stats)
for each in batch_size:
model = InitNN(num_inputs, d_num_hiddens, num_outputs)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, d_eps,
d_mom, num_epochs, each)
Save('results/batch_size_' + str(each) + '_nn.epz',stats)
for each in num_hiddens:
model = InitNN(num_inputs, each, num_outputs)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, d_eps,
0.9, num_epochs, d_batch_size)
Save('results/hidden_' + str(each[0]) + '_' + str(each[1]) + '_nn.epz',stats)
model = InitNN(num_inputs, d_num_hiddens, num_outputs)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, d_eps,
d_mom, num_epochs, d_batch_size)
Save('results/normal_nn.epz',stats)'''
"""Trains a NN."""
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [8, 16] #[16, 32]
eps = 0.01
momentum = 0.0
num_epochs = 200
batch_size = 10
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
model, stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size)
# Uncomment if you wish to save the model.
# Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
def main():
"""Trains a NN."""
#model_fname = 'nn_model.npz'
#stats_fname = 'nn_stats.npz'
#model_fname = 'nn_model_32_eps1.npz'
#stats_fname = 'nn_stats_32_eps1.npz'
#model_fname = 'nn_model_32_eps10.npz'
#stats_fname = 'nn_stats_32_eps10.npz'
#model_fname = 'nn_model_32_mom09.npz'
#stats_fname = 'nn_stats_32_mom09.npz'
#model_fname = 'nn_model_32_batch1000.npz'
#stats_fname = 'nn_stats_32_batch1000.npz'
model_fname = 'nn_model_33_50_100.npz'
stats_fname = 'nn_stats_33_50_100.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [2, 4]
eps = 0.01
momentum = 0.0
num_epochs = 1000
batch_size = 100
# Input-output dimensions.
#num_inputs = 2304
#num_outputs = 7
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
#x = np.random.rand(10, 48 * 48) * 0.1
#CheckGrad(model, NNForward, NNBackward, 'W3', x)
#CheckGrad(model, NNForward, NNBackward, 'b3', x)
#CheckGrad(model, NNForward, NNBackward, 'W2', x)
#CheckGrad(model, NNForward, NNBackward, 'b2', x)
#CheckGrad(model, NNForward, NNBackward, 'W1', x)
#CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
#stats = Train(model, NNForward, NNBackward, NNUpdate, eps,
# momentum, num_epochs, batch_size)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
def main():
"""Trains a NN."""
"""
configs = {
'default': {
'model_fname': '3.1_nn_model.npz',
'stats_fname': '3.1_nn_stats.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - eps=0.001': {
'model_fname': '3.2_nn_model_eps_0.001.npz',
'stats_fname': '3.2_nn_stats_eps_0.001.npz',
'eps': 0.001,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - eps=0.01': {
'model_fname': '3.2_nn_model_eps_0.01.npz',
'stats_fname': '3.2_nn_stats_eps_0.01.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - eps=0.1': {
'model_fname': '3.2_nn_model_eps_0.1.npz',
'stats_fname': '3.2_nn_stats_eps_0.1.npz',
'eps': 0.1,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - eps=0.5': {
'model_fname': '3.2_nn_model_eps_0.5.npz',
'stats_fname': '3.2_nn_stats_eps_0.5.npz',
'eps': 0.5,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - eps=1.0': {
'model_fname': '3.2_nn_model_eps_1.0.npz',
'stats_fname': '3.2_nn_stats_eps_1.0.npz',
'eps': 1.0,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - momentum=0.0': {
'model_fname': '3.2_nn_model_momentum_0.0.npz',
'stats_fname': '3.2_nn_stats_momentum_0.0.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - momentum=0.5': {
'model_fname': '3.2_nn_model_momentum_0.5.npz',
'stats_fname': '3.2_nn_stats_momentum_0.5.npz',
'eps': 0.01,
'momentum': 0.5,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - momentum=0.9': {
'model_fname': '3.2_nn_model_momentum_0.9.npz',
'stats_fname': '3.2_nn_stats_momentum_0.9.npz',
'eps': 0.01,
'momentum': 0.9,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - batch=1': {
'model_fname': '3.2_nn_model_batch_1.npz',
'stats_fname': '3.2_nn_stats_batch_1.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 1,
},
'3.2 - batch=10': {
'model_fname': '3.2_nn_model_batch_10.npz',
'stats_fname': '3.2_nn_stats_batch_10.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 10,
},
'3.2 - batch=100': {
'model_fname': '3.2_nn_model_batch_100.npz',
'stats_fname': '3.2_nn_stats_batch_100.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 100,
},
'3.2 - batch=500': {
'model_fname': '3.2_nn_model_batch_500.npz',
'stats_fname': '3.2_nn_stats_batch_500.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 500,
},
'3.2 - batch=1000': {
'model_fname': '3.2_nn_model_batch_1000.npz',
'stats_fname': '3.2_nn_stats_batch_1000.npz',
'eps': 0.01,
'momentum': 0.0,
'num_epochs': 1000,
'num_hiddens': [16, 32],
'batch_size': 1000,
},
'3.3 - hidden=8,16': {
'model_fname': '3.3_nn_model_hidden_8_16.npz',
'stats_fname': '3.3_nn_stats_hidden_8_16.npz',
'eps': 0.01,
'momentum': 0.9,
'num_epochs': 1000,
'num_hiddens': [8, 16],
'batch_size': 100,
},
'3.3 - hidden=32,64': {
'model_fname': '3.3_nn_model_hidden_32_64.npz',
'stats_fname': '3.3_nn_stats_hidden_32_64.npz',
'eps': 0.01,
'momentum': 0.9,
'num_epochs': 500,
'num_hiddens': [32, 64],
'batch_size': 100,
},
'3.3 - hidden=48,96': {
'model_fname': '3.3_nn_model_hidden_48_96.npz',
'stats_fname': '3.3_nn_stats_hidden_48_96.npz',
'eps': 0.01,
'momentum': 0.9,
'num_epochs': 300,
'num_hiddens': [48, 96],
'batch_size': 100,
},
}
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
for key, config in configs.items():
# Initialize model.
model = InitNN(num_inputs, config['num_hiddens'], num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
print(config)
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
trained_model, stats = Train(model, NNForward, NNBackward, NNUpdate, config['eps'],
config['momentum'], config['num_epochs'], config['batch_size'])
# Uncomment if you wish to save the model.
Save(config['model_fname'], trained_model)
# Uncomment if you wish to save the training statistics.
Save(config['stats_fname'], stats)
"""
model_fname = '3.4_nn_model_hidden_1_1.npz'
stats_fname = '3.4_nn_stats_hidden_1_1.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [1, 1]
eps = 0.01
momentum = 0.0
num_epochs = 1000
batch_size = 100
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
# Train model.
model, stats = Train(model, NNForward, NNBackward, NNUpdate, eps, momentum,
num_epochs, batch_size)
# Uncomment if you wish to save the model.
Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
DisplayPlot(stats['train_ce'],
stats['valid_ce'],
'Cross Entropy',
number=0,
time=1000)
DisplayPlot(stats['train_acc'],
stats['valid_acc'],
'Accuracy',
number=1,
time=1000)
def main():
"""Trains a NN."""
model_fname = 'nn_model.npz'
stats_fname = 'nn_stats.npz'
# Hyper-parameters. Modify them if needed.
num_hiddens = [16, 32] #[150,20] [6,2] [200, 140]
eps = [0.01] #[0.001, 0.01, 0.1, 0.5, 1.0]
momentum = [0.9] #[0.0, 0.45, 0.9]
num_epochs = 1000
batch_size = [100] #[1, 10, 100, 500, 1000]
# Input-output dimensions.
num_inputs = 2304
num_outputs = 7
# Initialize model.
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Uncomment to reload trained model here.
model = Load(
"C:\\Users\\Adam\\Documents\\Masters\\Machine Learning\\A2\\NN\\NN_Results\\0.01\\0.0\\100\\nn_model.npz"
)
# Check gradient implementation.
print('Checking gradients...')
x = np.random.rand(10, 48 * 48) * 0.1
CheckGrad(model, NNForward, NNBackward, 'W3', x)
CheckGrad(model, NNForward, NNBackward, 'b3', x)
CheckGrad(model, NNForward, NNBackward, 'W2', x)
CheckGrad(model, NNForward, NNBackward, 'b2', x)
CheckGrad(model, NNForward, NNBackward, 'W1', x)
CheckGrad(model, NNForward, NNBackward, 'b1', x)
print("passed grad check")
inputs_train, inputs_valid, inputs_test, target_train, target_valid, target_test = LoadData(
'../toronto_face.npz')
#print(inputs_test.shape)
Evaluate(inputs_test, target_test, model, NNForward, 1)
for e in eps:
for m in momentum:
for b in batch_size:
model = InitNN(num_inputs, num_hiddens, num_outputs)
# Train model.
dir = str(e) + "\\" + str(m) + "\\" + str(b) + "\\"
if os.path.exists(dir):
filelist = [
f for f in os.listdir(".") if f.endswith(".bak")
]
for f in filelist:
os.remove(f)
else:
os.makedirs(dir)
model, stats = Train(model, NNForward, NNBackward, NNUpdate, e,
m, num_epochs, b)
# Uncomment if you wish to save the model.
Save(dir + model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(dir + stats_fname, stats)
SavePlot(0, dir + "cross_entropy.png")
SavePlot(1, dir + "accuracy.png")
def main():
'''
Run on all combinations of hyper parameters
num_epochs = 1#30
filter_size = 5
num_filters_1 = 8
num_filters_2 = 16
li_num_filters_1 = [2,50,25,2,25]
li_num_filters_2 = [25,50,2,2,25]
eps = [.001,.01,.1,.5,1]
d_eps = .01
momentum = [0,.45,.9]
d_mom = 0.0
batch_size=[1,20,300,750,1000]
d_batch_size = 100
# Input-output dimensions.
num_channels = 1
num_outputs = 7
for i in range(10):
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
x = np.random.rand(10, 48, 48, 1) * 0.1
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
CheckGrad(model, CNNForward, CNNBackward, 'b1', x)
for each in eps:
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, each,
d_mom, num_epochs, d_batch_size)
Save('results/eps_' + str(each) + '_cnn.npz', stats)
for each in momentum:
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, d_eps,
each, num_epochs, d_batch_size)
Save('results/momentum_' + str(each) + '_cnn.npz',stats)
for each in batch_size:
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, d_eps,
d_mom, num_epochs, each)
Save('results/batch_size_' + str(each) + '_cnn.npz',stats)
for index,each in enumerate(li_num_filters_1):
model = InitCNN(num_channels, filter_size, each, li_num_filters_2[index],
num_outputs)
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, d_eps,
0.9, num_epochs, d_batch_size)
Save('results/num_filters_' + str(each) + '_' + str(li_num_filters_2[index]) + '_cnn.npz',stats)
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2,
num_outputs)
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, d_eps,
d_mom, num_epochs, d_batch_size)
Save('results/normal_cnn.npz',stats)'''
"""Trains a CNN."""
model_fname = 'exp2/cnn_nonclass_model.npz'
stats_fname = 'exp2/cnn_nonclass_stats.npz'
# Hyper-parameters. Modify them if needed.
eps = 0.05
momentum = 0.0
num_epochs = 60
filter_size = 5
num_filters_1 = 3
num_filters_2 = 32
num_filters_3 = 32
num_hiddens = 64
batch_size = 50
# Input-output dimensions.
num_channels = 3
num_outputs = 4
# Initialize model.
model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2, num_filters_3, num_hiddens,
num_outputs)
# Uncomment to reload trained model here.
# model = Load(model_fname)
# Check gradient implementation.
# Uncomment when gradient is implemented
print('Checking gradients...')
x = np.random.rand(10, 32, 32, 3) * 0.1
CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
# Train model.
model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps,
momentum, num_epochs, batch_size)
# Uncomment if you wish to save the model.
# Save(model_fname, model)
# Uncomment if you wish to save the training statistics.
Save(stats_fname, stats)
Save(model_fname, model)