def main(main_params, optimization_type="minibatch_sgd"):
### set the random seed ###
np.random.seed(int(main_params['random_seed']))
### data processing ###
Xtrain, Ytrain, Xval, Yval , _, _ = data_loader_mnist(dataset = main_params['input_file'])
N_train, d = Xtrain.shape
N_val, _ = Xval.shape
index = np.arange(10)
unique, counts = np.unique(Ytrain, return_counts=True)
counts = dict(zip(unique, counts)).values()
trainSet = DataSplit(Xtrain, Ytrain)
valSet = DataSplit(Xval, Yval)
model = dict()
num_L1 = 1000
num_L2 = 10
num_epoch = int(main_params['num_epoch'])
minibatch_size = int(main_params['minibatch_size'])
# optimization setting: _alpha for momentum, _lambda for weight decay
_learning_rate = float(main_params['learning_rate'])
_step = 10
_alpha = float(main_params['alpha'])
_lambda = float(main_params['lambda'])
_dropout_rate = float(main_params['dropout_rate'])
_activation = main_params['activation']
if _activation == 'relu':
act = relu
else:
act = tanh
# create objects (modules) from the module classes
model['L1'] = linear_layer(input_D = d, output_D = num_L1)
model['nonlinear1'] = act()
model['drop1'] = dropout(r = _dropout_rate)
model['L2'] = linear_layer(input_D = num_L1, output_D = num_L2)
model['loss'] = softmax_cross_entropy()
# Momentum
if _alpha > 0.0:
momentum = add_momentum(model)
else:
momentum = None
train_acc_record = []
val_acc_record = []
train_loss_record = []
val_loss_record = []
### run training and validation ###
for t in range(num_epoch):
print('At epoch ' + str(t + 1))
if (t % _step == 0) and (t != 0):
_learning_rate = _learning_rate * 0.1
idx_order = np.random.permutation(N_train)
train_acc = 0.0
train_loss = 0.0
train_count = 0
val_acc = 0.0
val_count = 0
val_loss = 0.0
for i in range(int(np.floor(N_train / minibatch_size))):
# get a mini-batch of data
x, y = trainSet.get_example(idx_order[i * minibatch_size : (i + 1) * minibatch_size])
### forward ###
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train = True)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
### backward ###
grad_d1 = model['L2'].backward(d1, grad_a2)
grad_h1 = model['drop1'].backward(h1, grad_d1)
grad_a1 = model['nonlinear1'].backward(a1, grad_h1)
######################################################################################
grad_x = model['L1'].backward(x, grad_a1)
### gradient_update ###
model = miniBatchGradientDescent(model, momentum, _lambda, _alpha, _learning_rate)
### Computing training accuracy and obj ###
for i in range(int(np.floor(N_train / minibatch_size))):
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train = False)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
train_loss += loss
train_acc += np.sum(predict_label(a2) == y)
train_count += len(y)
train_loss = train_loss
train_acc = train_acc / train_count
train_acc_record.append(train_acc)
train_loss_record.append(train_loss)
print('Training loss at epoch ' + str(t + 1) + ' is ' + str(train_loss))
print('Training accuracy at epoch ' + str(t + 1) + ' is ' + str(train_acc))
### Computing validation accuracy ###
for i in range(int(np.floor(N_val / minibatch_size))):
x, y = valSet.get_example(np.arange(i * minibatch_size, (i + 1) * minibatch_size))
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train = False)
a2 = model['L2'].forward(d1)
######################################################################################
loss = model['loss'].forward(a2, y)
val_loss += loss
val_acc += np.sum(predict_label(a2) == y)
val_count += len(y)
val_loss_record.append(val_loss)
val_acc = val_acc / val_count
val_acc_record.append(val_acc)
print('Validation accuracy at epoch ' + str(t + 1) + ' is ' + str(val_acc))
def main(main_params, optimization_type="minibatch_sgd"):
### set the random seed ###
np.random.seed(int(main_params['random_seed']))
### data processing ###
Xtrain, Ytrain, Xval, Yval, _, _ = data_loader_mnist(
dataset=main_params['input_file'])
N_train, d = Xtrain.shape
N_val, _ = Xval.shape
index = np.arange(10)
unique, counts = np.unique(Ytrain, return_counts=True)
counts = dict(zip(unique, counts)).values()
trainSet = DataSplit(Xtrain, Ytrain)
valSet = DataSplit(Xval, Yval)
### building/defining MLP ###
"""
In this script, we are going to build a MLP for a 10-class classification problem on MNIST.
The network structure is input --> linear --> relu --> dropout --> linear --> softmax_cross_entropy loss
the hidden_layer size (num_L1) is 1000
the output_layer size (num_L2) is 10
"""
model = dict()
num_L1 = 1000
num_L2 = 10
# experimental setup
num_epoch = int(main_params['num_epoch'])
minibatch_size = int(main_params['minibatch_size'])
# optimization setting: _alpha for momentum, _lambda for weight decay
_learning_rate = float(main_params['learning_rate'])
_step = 10
_alpha = float(main_params['alpha'])
_lambda = float(main_params['lambda'])
_dropout_rate = float(main_params['dropout_rate'])
_activation = main_params['activation']
if _activation == 'relu':
act = relu
else:
act = tanh
# create objects (modules) from the module classes
model['L1'] = linear_layer(input_D=d, output_D=num_L1)
model['nonlinear1'] = act()
model['drop1'] = dropout(r=_dropout_rate)
model['L2'] = linear_layer(input_D=num_L1, output_D=num_L2)
model['loss'] = softmax_cross_entropy()
# Momentum
if _alpha > 0.0:
momentum = add_momentum(model)
else:
momentum = None
train_acc_record = []
val_acc_record = []
train_loss_record = []
val_loss_record = []
### run training and validation ###
for t in range(num_epoch):
print('At epoch ' + str(t + 1))
if (t % _step == 0) and (t != 0):
_learning_rate = _learning_rate * 0.1
idx_order = np.random.permutation(N_train)
train_acc = 0.0
train_loss = 0.0
train_count = 0
val_acc = 0.0
val_count = 0
val_loss = 0.0
for i in range(int(np.floor(N_train / minibatch_size))):
# get a mini-batch of data
x, y = trainSet.get_example(idx_order[i * minibatch_size:(i + 1) *
minibatch_size])
### forward ###
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train=True)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
### backward ###
grad_a2 = model['loss'].backward(a2, y)
######################################################################################
# TODO: Call the backward methods of every layer in the model in reverse order
# We have given the first and last backward calls
# Do not modify them.
######################################################################################
grad_d1 = model['L2'].backward(d1, grad_a2)
grad_h1 = model['drop1'].backward(h1, grad_d1)
grad_a1 = model['nonlinear1'].backward(a1, grad_h1)
#raise NotImplementedError("Not Implemented BACKWARD PASS in main()")
######################################################################################
# NOTE: DO NOT MODIFY CODE BELOW THIS, until next TODO
######################################################################################
grad_x = model['L1'].backward(x, grad_a1)
### gradient_update ###
model = miniBatchGradientDescent(model, momentum, _lambda, _alpha,
_learning_rate)
### Computing training accuracy and obj ###
for i in range(int(np.floor(N_train / minibatch_size))):
x, y = trainSet.get_example(
np.arange(i * minibatch_size, (i + 1) * minibatch_size))
### forward ###
######################################################################################
# TODO: Call the forward methods of every layer in the model in order
# Check above forward code
# Make sure to keep train as False
######################################################################################
#raise NotImplementedError("Not Implemented COMPUTING TRAINING ACCURACY in main()")
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train=False)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
######################################################################################
# NOTE: DO NOT MODIFY CODE BELOW THIS, until next TODO
######################################################################################
loss = model['loss'].forward(a2, y)
train_loss += loss
train_acc += np.sum(predict_label(a2) == y)
train_count += len(y)
train_loss = train_loss
train_acc = train_acc / train_count
train_acc_record.append(train_acc)
train_loss_record.append(train_loss)
print('Training loss at epoch ' + str(t + 1) + ' is ' +
str(train_loss))
print('Training accuracy at epoch ' + str(t + 1) + ' is ' +
str(train_acc))
### Computing validation accuracy ###
for i in range(int(np.floor(N_val / minibatch_size))):
x, y = valSet.get_example(
np.arange(i * minibatch_size, (i + 1) * minibatch_size))
### forward ###
######################################################################################
# TODO: Call the forward methods of every layer in the model in order
# Check above forward code
# Make sure to keep train as False
######################################################################################
#raise NotImplementedError("Not Implemented COMPUTING VALIDATION ACCURACY in main()")
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train=False)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
######################################################################################
# NOTE: DO NOT MODIFY CODE BELOW THIS, until next TODO
######################################################################################
loss = model['loss'].forward(a2, y)
val_loss += loss
val_acc += np.sum(predict_label(a2) == y)
val_count += len(y)
val_loss_record.append(val_loss)
val_acc = val_acc / val_count
val_acc_record.append(val_acc)
print('Validation accuracy at epoch ' + str(t + 1) + ' is ' +
str(val_acc))
# save file
json.dump(
{
'train': train_acc_record,
'val': val_acc_record
},
open(
'MLP_lr' + str(main_params['learning_rate']) + '_m' +
str(main_params['alpha']) + '_w' + str(main_params['lambda']) +
'_d' + str(main_params['dropout_rate']) + '_a' +
str(main_params['activation']) + '.json', 'w'))
print('Finish running!')
return train_loss_record, val_loss_record
def main(main_params, optimization_type="minibatch_sgd"):
np.random.seed(int(main_params['random_seed']))
Xtrain, Ytrain, Xval, Yval, _, _ = data_loader_mnist(
dataset=main_params['input_file'])
N_train, d = Xtrain.shape
N_val, _ = Xval.shape
index = np.arange(10)
unique, counts = np.unique(Ytrain, return_counts=True)
counts = dict(zip(unique, counts)).values()
trainSet = DataSplit(Xtrain, Ytrain)
valSet = DataSplit(Xval, Yval)
model = dict()
num_L1 = 1000
num_L2 = 10
num_epoch = int(main_params['num_epoch'])
minibatch_size = int(main_params['minibatch_size'])
_learning_rate = float(main_params['learning_rate'])
_step = 10
_alpha = float(main_params['alpha'])
_lambda = float(main_params['lambda'])
_dropout_rate = float(main_params['dropout_rate'])
_activation = main_params['activation']
if _activation == 'relu':
act = relu
else:
act = tanh
model['L1'] = linear_layer(input_D=d, output_D=num_L1)
model['nonlinear1'] = act()
model['drop1'] = dropout(r=_dropout_rate)
model['L2'] = linear_layer(input_D=num_L1, output_D=num_L2)
model['loss'] = softmax_cross_entropy()
# Momentum
if _alpha > 0.0:
momentum = add_momentum(model)
else:
momentum = None
train_acc_record = []
val_acc_record = []
train_loss_record = []
val_loss_record = []
for t in range(num_epoch):
print('At epoch ' + str(t + 1))
if (t % _step == 0) and (t != 0):
_learning_rate = _learning_rate * 0.1
idx_order = np.random.permutation(N_train)
train_acc = 0.0
train_loss = 0.0
train_count = 0
val_acc = 0.0
val_count = 0
val_loss = 0.0
for i in range(int(np.floor(N_train / minibatch_size))):
# get a mini-batch of data
x, y = trainSet.get_example(idx_order[i * minibatch_size:(i + 1) *
minibatch_size])
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train=True)
a2 = model['L2'].forward(d1)
a2 = model['L2'].forward(h1)
loss = model['loss'].forward(a2, y)
grad_a2 = model['loss'].backward(a2, y)
l2_backward = model['L2'].backward(d1, grad_a2)
drop1_backward = model['drop1'].backward(h1, l2_backward)
nonlinear1_backward = model['nonlinear1'].backward(
a1, drop1_backward)
grad_x = model['L1'].backward(x, nonlinear1_backward)
model = miniBatchGradientDescent(model, momentum, _lambda, _alpha,
_learning_rate)
for i in range(int(np.floor(N_train / minibatch_size))):
x, y = trainSet.get_example(
np.arange(i * minibatch_size, (i + 1) * minibatch_size))
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train=False)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
loss = model['loss'].forward(a2, y)
train_loss += loss
train_acc += np.sum(predict_label(a2) == y)
train_count += len(y)
train_loss = train_loss
train_acc = train_acc / train_count
train_acc_record.append(train_acc)
train_loss_record.append(train_loss)
print('Training loss at epoch ' + str(t + 1) + ' is ' +
str(train_loss))
print('Training accuracy at epoch ' + str(t + 1) + ' is ' +
str(train_acc))
for i in range(int(np.floor(N_val / minibatch_size))):
x, y = valSet.get_example(
np.arange(i * minibatch_size, (i + 1) * minibatch_size))
a1 = model['L1'].forward(x)
h1 = model['nonlinear1'].forward(a1)
d1 = model['drop1'].forward(h1, is_train=False)
a2 = model['L2'].forward(d1)
loss = model['loss'].forward(a2, y)
loss = model['loss'].forward(a2, y)
val_loss += loss
val_acc += np.sum(predict_label(a2) == y)
val_count += len(y)
val_loss_record.append(val_loss)
val_acc = val_acc / val_count
val_acc_record.append(val_acc)
print('Validation accuracy at epoch ' + str(t + 1) + ' is ' +
str(val_acc))
json.dump(
{
'train': train_acc_record,
'val': val_acc_record
},
open(
'MLP_lr' + str(main_params['learning_rate']) + '_m' +
str(main_params['alpha']) + '_w' + str(main_params['lambda']) +
'_d' + str(main_params['dropout_rate']) + '_a' +
str(main_params['activation']) + '.json', 'w'))
print('Finish running!')
return train_loss_record, val_loss_record