def main():
''' Main function - fill this in '''
## %% =========== Part 1: Loading and Visualizing Data =============
#%
# read Kaggle data, and display summary of it.
m, n, X, y = getKaggleTrainingData()
# display a sample of the data & corresponding labels
displaySampleData(X, y)
# Partition the Kaggle training data
X_train, y_train, X_cv, y_cv, X_test, y_test = partitionData(X, y)
# jkm - debug
print '\n Kaggle Data Partitioned into Train, cv, test'
print 'Labels :', np.arange(10)
print 'y :', np.bincount(np.hstack(y))
print 'y_train:', np.bincount(np.hstack(y_train))
print 'y_cv :', np.bincount(np.hstack(y_cv))
print 'y_test :', np.bincount(np.hstack(y_test))
print 'Totals :', np.bincount(np.hstack(y_train)) + np.bincount(np.hstack(y_cv)) + np.bincount(np.hstack(y_test))
# features
input_layer_size = n
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
#%% ================= Part 2: Train the NN =================
# JKM - I will loop through and find the best results
# and store the NN
# but for now just continue
#
# jkmm - put inline again
# Callback for displaying the cost at the end of each iteration
class Callback(object):
def __init__(self, input_layer_size, hidden_layer_size, num_labels,
X_cv, y_cv, _lambda, costFunc):
self.it = 0
self.input_layer_size = input_layer_size
self.hidden_layer_size = hidden_layer_size
self.num_labels = num_labels
self.X_cv = X_cv
self.y_cv = y_cv
self._lambda = _lambda
self.costFunc = costFunc
def __call__(self, p ):
self.it += 1
J_train = self.costFunc(p)[0]
# Calculate the cv cost every 10 iterations
if (self.it % 10 == 0):
J_cv, _ = nnCostFunction(p, self.input_layer_size, self.hidden_layer_size,
self.num_labels, self.X_cv, self.y_cv, self._lambda)
diff = np.abs(J_train - J_cv)
print "Iter %5d | J_train: %e | J_cv: %e | Diff: %e" % (self.it, J_train, J_cv, diff)
else:
print "Iter %5d | J_train: %e" % (self.it, J_train)
''' put back the code into inline on this script
for some speed improvement
Theta1, Theta2 = trainNN(input_layer_size, hidden_layer_size, num_labels, _lambda,
X_train, y_train, X_cv, y_cv)
'''
# Step 1: Initializing Parameters
# initial_nn_params = initializeNN(input_layer_size, hidden_layer_size, num_labels)
# make inline again
print('\nInitializing Neural Network Parameters ...\n')
initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size)
initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels)
#% Unroll parameters
initial_nn_params = np.hstack(( initial_Theta1.ravel(order = 'F'),
initial_Theta2.ravel(order = 'F')))
options = {'maxiter': MAXITER} # jkm - need to think about finding this best value
best_lambda = np.core.numeric.NaN
best_acc = 0
########### Start of Loop #########
# Loop to find best _lambda, using train data to tarin, and cv data to evaluate.
# started this loop , stopped and restarted to make lambda a float [0.1, 0.5, 1, 3, 5]
for _lambda in [1.0, 2.0, 3.0, 5.0]: # this is to be calculated later
## jkm - need to make lambdas as floats.
# actually _lambda gets turned into a float in teh cost function anyway.
_lambda = float(_lambda)
# Step 2: Training NN
print ('\nTraining Neural Network... \n')
print('\n Parms: Hidden Layer Units: {0} Max Iters: {1} Lambda: {2} \n'.format(
hidden_layer_size, MAXITER, _lambda))
#% Create "short hand" for the cost function to be minimized
#% Now, costFunction is a function that takes in only one argument (the
#% neural network parameters)
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
num_labels, X_train, y_train, _lambda)
'''
NOTES: Call scipy optimize minimize function
method : str or callable, optional Type of solver.
CG -> Minimization of scalar function of one or more variables
using the conjugate gradient algorithm.
jac : bool or callable, optional Jacobian (gradient) of objective function.
Only for CG, BFGS, Newton-CG, L-BFGS-B, TNC, SLSQP, dogleg, trust-ncg.
If jac is a Boolean and is True, fun is assumed to return the gradient
along with the objective function. If False, the gradient will be
estimated numerically. jac can also be a callable returning the
gradient of the objective. In this case, it must accept the same
arguments as fun.
callback : callable, optional. Called after each iteration, as callback(xk),
where xk is the current parameter vector.
'''
result = sci.minimize(costFunc, initial_nn_params, method='CG',
jac=True, options=options,
callback=Callback(input_layer_size, hidden_layer_size,
num_labels, X_cv, y_cv, _lambda, costFunc))
nn_params = result.x
cost = result.fun
# Debug statement
print('\n Results from minimizer function Success: {0} \n {1} '.format(
result.success, result.message))
#% Obtain Theta1 and Theta2 back from nn_params
Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],
(hidden_layer_size, (input_layer_size + 1)),
order = 'F')
Theta2 = np.reshape(nn_params[hidden_layer_size * (input_layer_size + 1):],
(num_labels, (hidden_layer_size + 1)),
order = 'F')
# Pause
#print("Program paused. Press Ctrl-D to continue.\n")
#code.interact(local=dict(globals(), **locals()))
#print(" ... continuing\n ")
# jkmm - pauses in here
# visualizeNN(Theta1, hidden_layer_size, MAXITER, _lambda)
#%% ================= Part 3 Predict =================
# it to get a good _lambda and other parms
pred = predict(Theta1, Theta2, X_cv)
# display a sample of the data & corresponding predicted labels
# displaySampleData(X_cv, np.vstack(pred)) # jkmm - comment out
''' above displaySample does this so comment out for now
# print out what the images are predicted to be
print('Selecting random examples of the data to display and how they are predicted.\n')
print(pred[sel].reshape(10, 10))
# display the sample images
displayData(images)
'''
# JKM - my array was column stacked - don't understand why this works
pp = np.row_stack(pred)
accuracy = np.mean(np.double(pp == y_cv)) * 100
print('\n Cross Valid Set Accuracy: {0} \n'.format(accuracy))
print('\n Parms: Hidden Layer Units: {0} Max Iters: {1} Lambda: {2} \n'.format(
hidden_layer_size, MAXITER, _lambda))
# Create a filname to use to write the results
fn = 'HU_{0}_MaxIter_{1}_Lambda_{2}_PredAcc_{3}'.format(
hidden_layer_size, MAXITER, _lambda, accuracy)
# Capture the Thetas
writeLearnedTheta(Theta1, 'Theta1_' + fn)
writeLearnedTheta(Theta2, 'Theta2_' + fn)
if (accuracy > best_acc):
best_acc = accuracy
best_lambda = _lambda
print 'updating _lambda & best_acc'
# end of _lambda loop
print 'end of _lambda loop'
# Pause
print("Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
'''
def main(argv=None): # pylint: disable=unused-argument
if FLAGS.self_test:
print 'Running self-test.'
train_data, train_labels = fake_data(256)
validation_data, validation_labels = fake_data(16)
test_data, test_labels = fake_data(256)
num_epochs = 1
else:
# Get the data. #### MNIST DATA ####
''' JKM - added tf_
tf_train_data_filename = maybe_download('train-images-idx3-ubyte.gz')
tf_train_labels_filename = maybe_download('train-labels-idx1-ubyte.gz')
tf_test_data_filename = maybe_download('t10k-images-idx3-ubyte.gz')
tf_test_labels_filename = maybe_download('t10k-labels-idx1-ubyte.gz')
# Extract it into numpy arrays.
# jkm - rename to tf_
tf_train_data = extract_data(tf_train_data_filename, 60000)
tf_train_labels = extract_labels(tf_train_labels_filename, 60000)
tf_test_data = extract_data(tf_test_data_filename, 10000)
tf_test_labels = extract_labels(tf_test_labels_filename, 10000)
# Generate a validation set JKM - adding tf_
tf_validation_data = tf_train_data[:VALIDATION_SIZE, :, :, :]
tf_validation_labels = tf_train_labels[:VALIDATION_SIZE]
tf_train_data = tf_train_data[VALIDATION_SIZE:, :, :, :]
tf_train_labels = tf_train_labels[VALIDATION_SIZE:]
'''
### KAGGLE DATA #####
## JKM - get Kaggle comp data & convert to 4D tensor data
m, n, X, y = getKaggleTrainingData(scale = 2.0, stats = False)
# JKM - display a sample of the data & corresponding labels
#displaySampleData(X, y)
# JKM- partition my X data set into 3 sets, X, CV, TEST
# Partition the Kaggle training data, randomly
X_train, y_train, X_cv, y_cv, X_test, y_test = partitionData(X, y)
# JKM-Convert to 4D tensor data and 1-hot matrix labels
train_data = convert_data(X_train, X_train.shape[0])
train_labels = convert_labels(y_train)
validation_data = convert_data(X_cv, X_cv.shape[0])
validation_labels = convert_labels(y_cv)
test_data = convert_data(X_test, X_test.shape[0])
test_labels = convert_labels(y_test)
# JKM- Get the Kaggle test data
# get the data to do the final test.
m_KaggleTest, n_KaggleTest, X_KaggleTest = getKaggleTestData(scale = 2)
kaggle_test_data = convert_data(X_KaggleTest, m_KaggleTest)
num_epochs = NUM_EPOCHS
train_size = train_labels.shape[0]
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
train_data_node = tf.placeholder(
tf.float32,
shape=(BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
train_labels_node = tf.placeholder(tf.float32,
shape=(BATCH_SIZE, NUM_LABELS))
# For the validation and test data, we'll just hold the entire dataset in
# one constant node.
validation_data_node = tf.constant(validation_data)
test_data_node = tf.constant(test_data)
# jkm - create a node for the kaggle data
# create 2 types of nodes, one regular and one batch
kaggle_test_data_node = tf.constant(kaggle_test_data)
batch_kaggle_test_data_node = tf.placeholder(
tf.float32,
shape=(KAGGLE_BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS))
# The variables below hold all the trainable weights. They are passed an
# initial value which will be assigned when when we call:
# {tf.initialize_all_variables().run()}
conv1_weights = tf.Variable(
tf.truncated_normal([5, 5, NUM_CHANNELS, 32], # 5x5 filter, depth 32.
stddev=0.1,
seed=SEED))
conv1_biases = tf.Variable(tf.zeros([32]))
conv2_weights = tf.Variable(
tf.truncated_normal([5, 5, 32, 64],
stddev=0.1,
seed=SEED))
conv2_biases = tf.Variable(tf.constant(0.1, shape=[64]))
fc1_weights = tf.Variable( # fully connected, depth 512.
tf.truncated_normal([IMAGE_SIZE / 4 * IMAGE_SIZE / 4 * 64, 512],
stddev=0.1,
seed=SEED))
fc1_biases = tf.Variable(tf.constant(0.1, shape=[512]))
fc2_weights = tf.Variable(
tf.truncated_normal([512, NUM_LABELS],
stddev=0.1,
seed=SEED))
fc2_biases = tf.Variable(tf.constant(0.1, shape=[NUM_LABELS]))
# We will replicate the model structure for the training subgraph, as well
# as the evaluation subgraphs, while sharing the trainable parameters.
def model(data, train=False):
"""The Model definition."""
# 2D convolution, with 'SAME' padding (i.e. the output feature map has
# the same size as the input). Note that {strides} is a 4D array whose
# shape matches the data layout: [image index, y, x, depth].
conv = tf.nn.conv2d(data,
conv1_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non-linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv, conv1_biases))
# Max pooling. The kernel size spec {ksize} also follows the layout of
# the data. Here we have a pooling window of 2, and a stride of 2.
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
conv = tf.nn.conv2d(pool,
conv2_weights,
strides=[1, 1, 1, 1],
padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, conv2_biases))
pool = tf.nn.max_pool(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# Reshape the feature map cuboid into a 2D matrix to feed it to the
# fully connected layers.
pool_shape = pool.get_shape().as_list()
reshape = tf.reshape(
pool,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3]])
# Fully connected layer. Note that the '+' operation automatically
# broadcasts the biases.
hidden = tf.nn.relu(tf.matmul(reshape, fc1_weights) + fc1_biases)
# Add a 50% dropout during training only. Dropout also scales
# activations such that no rescaling is needed at evaluation time.
if train:
hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
return tf.matmul(hidden, fc2_weights) + fc2_biases
# Training computation: logits + cross-entropy loss.
logits = model(train_data_node, True)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits, train_labels_node))
# L2 regularization for the fully connected parameters.
regularizers = (tf.nn.l2_loss(fc1_weights) + tf.nn.l2_loss(fc1_biases) +
tf.nn.l2_loss(fc2_weights) + tf.nn.l2_loss(fc2_biases))
# Add the regularization term to the loss.
loss += 5e-4 * regularizers
# Optimizer: set up a variable that's incremented once per batch and
# controls the learning rate decay.
batch = tf.Variable(0)
# Decay once per epoch, using an exponential schedule starting at 0.01.
learning_rate = tf.train.exponential_decay(
0.01, # Base learning rate.
batch * BATCH_SIZE, # Current index into the dataset.
train_size, # Decay step.
0.95, # Decay rate.
staircase=True)
# Use simple momentum for the optimization.
optimizer = tf.train.MomentumOptimizer(learning_rate,
0.9).minimize(loss,
global_step=batch)
# Predictions for the minibatch, validation set and test set.
train_prediction = tf.nn.softmax(logits)
# We'll compute them only once in a while by calling their {eval()} method.
validation_prediction = tf.nn.softmax(model(validation_data_node))
test_prediction = tf.nn.softmax(model(test_data_node))
# jkm - set the prediction. debug - I have 2, one regular and one batch
kaggle_test_prediction = tf.nn.softmax(model(kaggle_test_data_node))
batch_kaggle_test_prediction = tf.nn.softmax(model(batch_kaggle_test_data_node))
# Create a local session to run this computation.
with tf.Session() as s:
# Run all the initializers to prepare the trainable parameters.
tf.initialize_all_variables().run()
print 'Initialized!'
# Loop through training steps.
for step in xrange(int(num_epochs * train_size / BATCH_SIZE)):
# Compute the offset of the current minibatch in the data.
# Note that we could use better randomization across epochs.
offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)
batch_data = train_data[offset:(offset + BATCH_SIZE), :, :, :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
# This dictionary maps the batch data (as a numpy array) to the
# node in the graph is should be fed to.
feed_dict = {train_data_node: batch_data,
train_labels_node: batch_labels}
# Run the graph and fetch some of the nodes.
_, l, lr, predictions = s.run(
[optimizer, loss, learning_rate, train_prediction],
feed_dict=feed_dict)
if step % 100 == 0:
print 'Epoch %.2f' % (float(step) * BATCH_SIZE / train_size)
print 'Minibatch loss: %.3f, learning rate: %.6f' % (l, lr)
print 'Minibatch error: %.1f%%' % error_rate(predictions,
batch_labels)
print 'Validation error: %.1f%%' % error_rate(
validation_prediction.eval(), validation_labels)
sys.stdout.flush()
# Finally print the result!
test_error = error_rate(test_prediction.eval(), test_labels)
print 'Test error: %.1f%%' % test_error
if FLAGS.self_test:
print 'test_error', test_error
assert test_error == 0.0, 'expected 0.0 test_error, got %.2f' % (
test_error,)
# jkm - Get the Kaggle test results, need to use the kaggle batch node/model
#kaggle_pred = kaggle_test_prediction.eval()
#KAGGLE_BATCH_SIZE = 100
kaggle_result = numpy.zeros(m_KaggleTest)
batch_test_size = m_KaggleTest # jkm - don't forget the last less then 64
for step in xrange(batch_test_size / KAGGLE_BATCH_SIZE):
offset = step * KAGGLE_BATCH_SIZE
k_batch_data = kaggle_test_data[offset:(offset + KAGGLE_BATCH_SIZE), :, :, :]
feed_dict = { batch_kaggle_test_data_node: k_batch_data }
(batch_predictions,) = s.run([batch_kaggle_test_prediction], feed_dict=feed_dict)
kaggle_result[offset:(offset + KAGGLE_BATCH_SIZE)] = numpy.argmax(batch_predictions, 1)
# jkm - Display some of the results
displaySampleData(X_KaggleTest, numpy.vstack(kaggle_result))
# jkm - write the results
# Create a filname to use to write the results
fn = 'CNN_Num_Epochs_{0}_Last_LR_{1}_Test_Error_{2}'.format(
num_epochs, lr, test_error)
writeKaggleTestResults(fn, kaggle_result)
# Pause - jkm
print("9__Program paused. Press Ctrl-D to continue.\n")
code.interact(local=dict(globals(), **locals()))
print(" ... continuing\n ")
