def buildAndTrainModel(hiddenSize=100, learnRate=0.5, learnRateDecay=1.0, optimizeSteps = 1000, stochastic=False,
batchSize=128, trainSubsetSize=100000, reportEvery=100):
graph = tf.Graph()
global train_dataset, train_labels
# truncate train dataset, only effective from non-stochastic training
if trainSubsetSize:
train_dataset = train_dataset[:trainSubsetSize]
train_labels = train_labels[:trainSubsetSize]
## build graph
with graph.as_default():
# setup model
if stochastic:
train = tf.placeholder(np.float32, (batchSize, exampleWidth))
trainLabels = tf.placeholder(np.float32, (batchSize, num_labels))
else:
train = tf.constant(train_dataset)
trainLabels = tf.constant(train_labels)
weights1 = tf.Variable(tf.truncated_normal((exampleWidth, hiddenSize)))
bias1 = tf.Variable(tf.zeros(hiddenSize))
weights2 = tf.Variable(tf.truncated_normal((hiddenSize, num_labels)))
bias2 = tf.Variable(tf.zeros(num_labels))
def nnOutput(input, raw = False):
''' create a variable representing network output for given input variable '''
out = tf.nn.relu(tf.matmul(input, weights1)+bias1)
out = tf.matmul(out, weights2)+bias2
if not raw: out = tf.nn.softmax(out)
return out
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=trainLabels, logits=nnOutput(train, raw=True)))
optimizer = tf.train.GradientDescentOptimizer(learnRate).minimize(loss)
# setup classification performance indicators
trainOutput = nnOutput(train)
valid, test = tf.constant(valid_dataset), tf.constant(test_dataset)
validOutput, testOutput = nnOutput(valid), nnOutput(test)
## run model optimization
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
for s in range(optimizeSteps):
if stochastic:
start = (s*batchSize) % (train_dataset.shape[0] - batchSize)
feeds = {train: train_dataset[start:start+batchSize],
trainLabels: train_labels[start:start+batchSize]}
trlabels = train_labels[start:start+batchSize]
else:
feeds=None
trlabels = train_labels
# execute GDesc update
_, train_res, valid_res, test_res = \
session.run([optimizer, trainOutput, validOutput, testOutput],
feed_dict=feeds)
if s % reportEvery == 0:
print('step %d: train_acc %.4f, valid_acc %.4f' % \
(s, accuracy(train_res, trlabels), accuracy(valid_res, valid_labels)))
print('TEST accuracy: %.4f' % accuracy(test_res, test_labels))
def buildAndTrainModel(layers=[500],
learnRate=0.01,
momentum=0.95,
dropout=0.5,
decay=0.99,
decayStart=100,
optimizeSteps=1000,
stochastic=True,
batchSize=128,
trainSubsetSize=100000,
reportEvery=100):
graph = tf.Graph()
global train_dataset, train_labels
# truncate train dataset, only effective for non-stochastic training
if trainSubsetSize:
train_dataset = train_dataset[:trainSubsetSize]
train_labels = train_labels[:trainSubsetSize]
## build graph
with graph.as_default():
# tf.add_check_numerics_ops()
# setup model
trainFull = tf.constant(train_dataset)
if stochastic:
train = tf.placeholder(np.float32, (batchSize, exampleWidth))
trainLabels = tf.placeholder(np.float32, (batchSize, num_labels))
else:
train = trainFull
trainLabels = tf.constant(train_labels)
network = MultilayerNN(input=train,
inputSize=exampleWidth,
outputSize=num_labels,
hiddenLayers=layers,
dropout=dropout)
trainOutput = network.out(train, True, activation=None)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=trainLabels,
logits=trainOutput))
# setup learning rate decay and optmization
global_step = tf.Variable(0, trainable=False)
learning_rate = tf.train.exponential_decay(learnRate,
global_step,
decay_steps=decayStart,
decay_rate=decay,
staircase=True)
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum).minimize(
loss, global_step)
# setup classification performance indicators
trainOutput = network.out(train)
fullTrainOutput = network.out(trainFull)
valid, test = tf.constant(valid_dataset), tf.constant(test_dataset)
validOutput, testOutput = network.out(valid), network.out(test)
## run model optimization
from notmnist.utils import accuracy
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
for s in range(optimizeSteps):
if stochastic:
start = (s * batchSize) % (train_dataset.shape[0] - batchSize)
feeds = {
train: train_dataset[start:start + batchSize],
trainLabels: train_labels[start:start + batchSize]
}
trlabels = train_labels[start:start + batchSize]
else:
feeds = None
trlabels = train_labels
# execute GDesc update
session.run([optimizer], feed_dict=feeds)
if s % reportEvery == 0:
lrate, train_res, full_train_res, valid_res, test_res = \
session.run([learning_rate, trainOutput, fullTrainOutput, validOutput, testOutput],
feed_dict=feeds)
print('learning rate %.4f' % lrate)
print('step %d: batch_train_acc %.4f, full_train_acc %.4f, valid_acc %.4f, test_acc %.4f' % \
(s, accuracy(train_res, trlabels), accuracy(full_train_res, train_labels),
accuracy(valid_res, valid_labels), accuracy(test_res, test_labels)))
print('TEST accuracy: %.4f' % accuracy(test_res, test_labels))
def maxpoolConvNet(batch_size=16, patch_size=5, depth=16, num_hidden=64):
graph = tf.Graph()
with graph.as_default():
# Input data.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, image_size,
image_size, num_channels))
tf_train_labels = tf.placeholder(tf.float32,
shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.
layer1_weights = tf.Variable(
tf.truncated_normal([patch_size, patch_size, num_channels, depth],
stddev=0.1))
layer1_biases = tf.Variable(tf.zeros([depth]))
layer2_weights = tf.Variable(
tf.truncated_normal([patch_size, patch_size, depth, depth],
stddev=0.1))
layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))
layer3_weights = tf.Variable(
tf.truncated_normal(
[image_size // 4 * image_size // 4 * depth, num_hidden],
stddev=0.1))
layer3_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))
layer4_weights = tf.Variable(
tf.truncated_normal([num_hidden, num_labels], stddev=0.1))
layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))
# Model.
def model(data):
conv = tf.nn.conv2d(data,
layer1_weights, [1, 1, 1, 1],
padding='SAME')
hidden = tf.nn.max_pool(conv + layer1_biases, [1, 2, 2, 1],
[1, 2, 2, 1],
padding='SAME')
hidden = tf.nn.relu(hidden)
conv = tf.nn.conv2d(hidden,
layer2_weights, [1, 1, 1, 1],
padding='SAME')
hidden = tf.nn.max_pool(conv + layer2_biases, [1, 2, 2, 1],
[1, 2, 2, 1],
padding='SAME')
hidden = tf.nn.relu(hidden)
shape = hidden.get_shape().as_list()
reshape = tf.reshape(hidden,
[shape[0], shape[1] * shape[2] * shape[3]])
hidden = tf.nn.relu(
tf.matmul(reshape, layer3_weights) + layer3_biases)
return tf.matmul(hidden, layer4_weights) + layer4_biases
# Training computation.
logits = model(tf_train_dataset)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels,
logits=logits))
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(model(tf_valid_dataset))
test_prediction = tf.nn.softmax(model(tf_test_dataset))
num_steps = 1001
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
print('Initialized')
for step in range(num_steps):
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
batch_data = train_dataset[offset:(offset + batch_size), :, :, :]
batch_labels = train_labels[offset:(offset + batch_size), :]
feed_dict = {
tf_train_dataset: batch_data,
tf_train_labels: batch_labels
}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 50 == 0):
print('Minibatch loss at step %d: %f' % (step, l))
print('Minibatch accuracy: %.1f%%' %
accuracy(predictions, batch_labels))
print('Validation accuracy: %.1f%%' %
accuracy(valid_prediction.eval(), valid_labels))
print('Test accuracy: %.1f%%' %
accuracy(test_prediction.eval(), test_labels))
def buildAndTrainModel():
graph = tf.Graph()
with graph.as_default():
# Input data.
# Load the training, validation and test data into constants that are
# attached to the graph.
tf_train_dataset = tf.constant(train_dataset[:train_subset, :])
tf_train_labels = tf.constant(train_labels[:train_subset])
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.
# These are the parameters that we are going to be training. The weight
# matrix will be initialized using random values following a (truncated)
# normal distribution. The biases get initialized to zero.
weights = tf.Variable(
tf.truncated_normal([image_size * image_size, num_labels]))
biases = tf.Variable(tf.zeros([num_labels]))
# Training computation.
# We multiply the inputs with the weight matrix, and add biases. We compute
# the softmax and cross-entropy (it's one operation in TensorFlow, because
# it's very common, and it can be optimized). We take the average of this
# cross-entropy across all training examples: that's our loss.
logits = tf.matmul(tf_train_dataset, weights) + biases
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels,
logits=logits))
# Optimizer.
# We are going to find the minimum of this loss using gradient descent.
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
# Predictions for the training, validation, and test data.
# These are not part of training, but merely here so that we can report
# accuracy figures as we train.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf_valid_dataset, weights) + biases)
test_prediction = tf.nn.softmax(
tf.matmul(tf_test_dataset, weights) + biases)
num_steps = 801
with tf.Session(graph=graph) as session:
# This is a one-time operation which ensures the parameters get initialized as
# we described in the graph: random weights for the matrix, zeros for the
# biases.
tf.global_variables_initializer().run()
print('Initialized')
for step in range(num_steps):
# Run the computations. We tell .run() that we want to run the optimizer,
# and get the loss value and the training predictions returned as numpy
# arrays.
_, l, predictions = session.run(
[optimizer, loss, train_prediction])
if (step % 100 == 0):
print('Loss at step %d: %f' % (step, l))
print('Training accuracy: %.1f%%' %
accuracy(predictions, train_labels[:train_subset, :]))
# Calling .eval() on valid_prediction is basically like calling run(), but
# just to get that one numpy array. Note that it recomputes all its graph
# dependencies.
print('Validation accuracy: %.1f%%' %
accuracy(valid_prediction.eval(), valid_labels))
print('Test accuracy: %.1f%%' %
accuracy(test_prediction.eval(), test_labels))
def buildAndTrainModel():
graph = tf.Graph()
with graph.as_default():
# Input data. For the training data, we use a placeholder that will be fed
# at run time with a training minibatch.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size,
image_size * image_size))
tf_train_labels = tf.placeholder(tf.float32,
shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
# Variables.
weights = tf.Variable(
tf.truncated_normal([image_size * image_size, num_labels]))
biases = tf.Variable(tf.zeros([num_labels]))
# Training computation.
logits = tf.matmul(tf_train_dataset, weights) + biases
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels,
logits=logits))
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(
tf.matmul(tf_valid_dataset, weights) + biases)
test_prediction = tf.nn.softmax(
tf.matmul(tf_test_dataset, weights) + biases)
#num_steps = 3001
num_steps = 10001
from notmnist.utils import accuracy
with tf.Session(graph=graph) as session:
tf.global_variables_initializer().run()
print("Initialized")
for step in range(num_steps):
# Pick an offset within the training data, which has been randomized.
# Note: we could use better randomization across epochs.
offset = (step * batch_size) % (train_labels.shape[0] - batch_size)
# Generate a minibatch.
batch_data = train_dataset[offset:(offset + batch_size), :]
batch_labels = train_labels[offset:(offset + batch_size), :]
# Prepare a dictionary telling the session where to feed the minibatch.
# The key of the dictionary is the placeholder node of the graph to be fed,
# and the value is the numpy array to feed to it.
feed_dict = {
tf_train_dataset: batch_data,
tf_train_labels: batch_labels
}
_, l, predictions = session.run(
[optimizer, loss, train_prediction], feed_dict=feed_dict)
if (step % 500 == 0):
print("Minibatch loss at step %d: %f" % (step, l))
print("Minibatch accuracy: %.1f%%" %
accuracy(predictions, batch_labels))
print("Validation accuracy: %.1f%%" %
accuracy(valid_prediction.eval(), valid_labels))
print("Test accuracy: %.1f%%" %
accuracy(test_prediction.eval(), test_labels))