def train(conf, X_train, Y_train, X_devel, Y_devel):
"""Run training
Args:
conf: Configuration dictionary
X_train: numpy array of floats with shape [input dimension, number of train examples]
Y_train: numpy array of integers with shape [output dimension, number of train examples]
X_devel: numpy array of floats with shape [input dimension, number of devel examples]
Y_devel: numpy array of integers with shape [output dimension, number of devel examples]
Returns:
params: Dictionary with trained parameters
train_progress: Dictionary with progress data, to be used in visualization.
devel_progress: Dictionary with progress data, to be used in visualization.
"""
print("Run training")
# Preparation
num_examples_in_epoch = X_train.shape[1]
example_indices = np.arange(0, num_examples_in_epoch)
np.random.shuffle(example_indices)
# Initialisation
params = model.initialization(conf)
# For displaying training progress
train_steps = []
train_ccr = []
train_cost = []
devel_steps = []
devel_ccr = []
# Start training
step = 0
epoch = 0
num_correct_since_last_check = 0
batch_start_index = 0
batch_end_index = conf['batch_size']
print("Number of training examples in one epoch: ", num_examples_in_epoch)
print("Start training")
while True:
start_time = time.time()
batch_indices = get_batch_indices(example_indices, batch_start_index,
batch_end_index)
X_batch = X_train[:, batch_indices]
Y_batch = model.one_hot(Y_train[batch_indices],
conf['output_dimension'])
Y_proposal, features = model.forward(conf,
X_batch,
params,
is_training=True)
print("Finish Foward")
cost_value, num_correct = model.cross_entropy_cost(Y_proposal, Y_batch)
#print("Finish Cross Entropy")
grad_params = model.backward(conf, Y_proposal, Y_batch, params,
features)
print("Finish Backward")
params = model.gradient_descent_update(conf, params, grad_params)
print("Finish Gradient Update")
print("Finish Training Number" + repr(step))
num_correct_since_last_check += num_correct
batch_start_index += conf['batch_size']
batch_end_index += conf['batch_size']
if batch_start_index >= num_examples_in_epoch:
epoch += 1
np.random.shuffle(example_indices)
batch_start_index = 0
batch_end_index = conf['batch_size']
step += 1
if np.isnan(cost_value):
print("ERROR: nan encountered")
break
if step % conf['train_progress'] == 0:
elapsed_time = time.time() - start_time
sec_per_batch = elapsed_time / conf['train_progress']
examples_per_sec = conf['batch_size'] * conf[
'train_progress'] / elapsed_time
ccr = num_correct / conf['batch_size']
running_ccr = (num_correct_since_last_check /
conf['train_progress'] / conf['batch_size'])
num_correct_since_last_check = 0
train_steps.append(step)
train_ccr.append(running_ccr)
train_cost.append(cost_value)
if conf['verbose']:
print(
"S: {0:>7}, E: {1:>4}, cost: {2:>7.4f}, CCR: {3:>7.4f} ({4:>6.4f}), "
"ex/sec: {5:>7.3e}, sec/batch: {6:>7.3e}".format(
step, epoch, cost_value, ccr, running_ccr,
examples_per_sec, sec_per_batch))
if step % conf['devel_progress'] == 0:
num_correct, num_evaluated = evaluate(conf, params, X_devel,
Y_devel)
devel_steps.append(step)
devel_ccr.append(num_correct / num_evaluated)
if conf['verbose']:
print(
"S: {0:>7}, Test on development set. CCR: {1:>5} / {2:>5} = {3:>6.4f}"
.format(step, num_correct, num_evaluated,
num_correct / num_evaluated))
if step >= conf['max_steps']:
print("Terminating training after {} steps".format(step))
break
train_progress = {
'steps': train_steps,
'ccr': train_ccr,
'cost': train_cost
}
devel_progress = {'steps': devel_steps, 'ccr': devel_ccr}
return params, train_progress, devel_progress
def main_test():
print("----------START OF TESTS-----------")
# Get configuration
conf = config()
################################### Task 1.1: Parameter initialization
from model import initialization
params = initialization(conf)
################################### Task 1.2: Forward propagation
# Import Activation functions [1.2a & 1.2b]
from model import activation
from model import softmax
# Test Activation functions
from tests import task_2a
from tests import task_2b
input_Z, expected_A = task_2a()
A = activation(input_Z, 'relu')
print('Activation valid?:',np.array_equal(expected_A, A))
input_Z, expected_S = task_2b()
S = softmax(input_Z)
print('Softmax valid?:',np.array_equal(np.round(expected_S,decimals=3), np.round(S,decimals=3)))
# Import Forward propagation [1.2c]
from model import forward
from tests import task_2c
### Test Forward propagation
conf, X_batch, params, expected_Z_1, expected_A_1, expected_Z_2, expected_Y_proposed = task_2c()
Y_proposed, features = forward(conf, X_batch, params, is_training=True)
print('feature Z_1 valid?:',np.array_equal(expected_Z_1, np.round(features['Z_1'],decimals=8)))
print('feature A_1 valid?:',np.array_equal(expected_A_1, np.round(features['A_1'],decimals=8)))
print('feature Z_2 valid?:',np.array_equal(expected_Z_2, np.round(features['Z_2'],decimals=8)))
print('proposed Y valid?:',np.array_equal(expected_Y_proposed, np.round(Y_proposed,decimals=8)))
################################### Task 1.3: Cross Entropy cost function
# Import Cost function
from model import cross_entropy_cost
from tests import task_3
### Test Cost function
Y_proposed, Y_batch, expected_cost_value, expected_num_correct = task_3()
cost_value, num_correct = cross_entropy_cost(Y_proposed, Y_batch)
print('Cost value valid?:',np.array_equal(np.round(expected_cost_value,decimals=4), np.round(cost_value,decimals=4)))
print('Number of succesess valid?:',np.array_equal(expected_num_correct, np.round(num_correct,decimals=4)))
################################### Task 1.4: Backward propagation
# Import Derivative of the activation function [1.4a]
from model import activation_derivative
from tests import task_4a
# Test Derivative of activation
input_Z, expected_dg_dz = task_4a()
dg_dz = activation_derivative(input_Z, "relu")
print('Derivative function valid?:',np.array_equal(expected_dg_dz, np.round(dg_dz,decimals=4)))
# Import Backward propagation [1.4b]
from model import backward
from tests import task_4b
# Test Backward propagation
(conf, Y_proposed, Y_batch, params, features,
expected_grad_W_1, expected_grad_b_1, expected_grad_W_2, expected_grad_b_2) = task_4b()
grad_params = backward(conf, Y_proposed, Y_batch, params, features)
print('Grad_W_1 valid?:',np.array_equal(np.round(expected_grad_W_1,decimals=4), np.round(grad_params["grad_W_1"],decimals=4)))
print('Grad_b_1 valid?:',np.array_equal(np.round(expected_grad_b_1,decimals=4), np.round(grad_params["grad_b_1"][:, np.newaxis],decimals=4)))
print('Grad_W_2 valid?:',np.array_equal(np.round(expected_grad_W_2,decimals=4), np.round(grad_params["grad_W_2"],decimals=4)))
print('Grad_b_2 valid?:',np.array_equal(np.round(expected_grad_b_2,decimals=4), np.round(grad_params["grad_b_2"][:, np.newaxis],decimals=4)))
################################### Task 1.5: Update parameters
# Import Update
from model import gradient_descent_update
from tests import task_5
# Test Update
(conf, params, grad_params,
expected_updated_W_1, expected_updated_b_1, expected_updated_W_2, expected_updated_b_2) = task_5()
updated_params = gradient_descent_update(conf, params, grad_params)
print('update of W_1 valid?:',np.array_equal(np.round(expected_updated_W_1,decimals=4), np.round(updated_params["W_1"],decimals=4)))
print('update of b_1 valid?:',np.array_equal(np.round(expected_updated_b_1,decimals=4), np.round(updated_params["b_1"],decimals=4)))
print('update of W_2 valid?:',np.array_equal(np.round(expected_updated_W_2,decimals=4), np.round(updated_params["W_2"],decimals=4)))
print('update of b_2 valid?:',np.array_equal(np.round(expected_updated_b_2,decimals=4), np.round(updated_params["b_2"],decimals=4)))
print("----------END OF TESTS-----------")