# Read in training data
with gzip.open(sys.argv[1]) as f:
data, labels = pickle.load(f)
# Starting values for w and b
w0 = np.zeros(data.shape[1])
b0 = 0
# Optimization
w, b = hw1.minimize_l2loss(data, labels, w0, b0, int(sys.argv[3]),
float(sys.argv[4]))
# Show the result of training
#print w,b
if w.size == 2:
plot_boundary_on_data.plot(data, labels, lambda x: hw1.f(x, w, b) > .5)
if w.size == 784: # special to MNIST
plt.imshow(w.reshape(28, 28))
plt.show()
# Test on test data
with gzip.open(sys.argv[2]) as f:
test_data, test_labels = pickle.load(f)
yhat = hw1.f(test_data, w, b) >= .5
print np.mean(
yhat == test_labels) * 100, "% of test examples classified correctly."
hw1.evaluate(hw1.f(test_data, w, b), test_labels)
def main(argv=None):
verbose = FLAGS.verbose
plot = FLAGS.plot
train_data_filename = FLAGS.train
train_data,train_labels = extract_data(train_data_filename)
train_labels[train_labels==0] = -1
train_size,num_features = train_data.shape
num_epochs = FLAGS.num_epochs
svmC = FLAGS.svmC
x = tf.placeholder("float", shape=[None, num_features])
y = tf.placeholder("float", shape=[None,1])
W = tf.Variable(tf.zeros([num_features,1]))
b = tf.Variable(tf.zeros([1]))
y_raw = tf.matmul(x,W) + b
# Optimization.
regularization_loss = 0.5*tf.reduce_sum(tf.square(W))
hinge_loss = tf.reduce_sum(tf.maximum(tf.zeros([BATCH_SIZE,1]),
1 - y*y_raw));
svm_loss = regularization_loss + svmC*hinge_loss;
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss)
# Evaluation.
predicted_class = tf.sign(y_raw);
correct_prediction = tf.equal(y,predicted_class)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
with tf.Session() as s:
tf.initialize_all_variables().run()
if verbose:
print ('Initialized!')
print ()
print ('Training.')
for step in xrange(num_epochs * train_size // BATCH_SIZE):
if verbose:
print (step),
offset = (step * BATCH_SIZE) % train_size
batch_data = train_data[offset:(offset + BATCH_SIZE), :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
train_step.run(feed_dict={x: batch_data, y: batch_labels})
print ('loss: ', svm_loss.eval(feed_dict={x: batch_data, y: batch_labels}))
if verbose and offset >= train_size-BATCH_SIZE:
print
if verbose:
print
print ('Weight matrix.')
print (s.run(W))
print
print ('Bias vector.')
print (s.run(b))
print
print ("Applying model to first test instance.")
print
print ("Accuracy on train:", accuracy.eval(feed_dict={x: train_data, y: train_labels}))
if plot:
eval_fun = lambda X: predicted_class.eval(feed_dict={x:X});
plot_boundary_on_data.plot(train_data, train_labels, eval_fun)
def main(argv=None):
# Be verbose?
verbose = FLAGS.verbose
# Plot?
plot = FLAGS.plot
# Get the data.
train_data_filename = FLAGS.train
test_data_filename = FLAGS.test
# Extract it into numpy arrays.
train_data,train_labels = extract_data(train_data_filename)
test_data, test_labels = extract_data(test_data_filename)
# Get the shape of the training data.
train_size,num_features = train_data.shape
# Get the number of epochs for training.
num_epochs = FLAGS.num_epochs
# Get the size of layer one.
num_hidden = FLAGS.num_hidden
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
x = tf.placeholder("float", shape=[None, num_features])
y_ = tf.placeholder("float", shape=[None, NUM_LABELS])
# For the test data, hold the entire dataset in one constant node.
test_data_node = tf.constant(test_data)
# Define and initialize the network.
# Initialize the hidden weights and biases.
w_hidden = init_weights(
[num_features, num_hidden],
'xavier',
xavier_params=(num_features, num_hidden))
b_hidden = init_weights([1,num_hidden],'zeros')
# The hidden layer.
hidden = tf.nn.tanh(tf.matmul(x,w_hidden) + b_hidden)
# Initialize the output weights and biases.
w_out = init_weights(
[num_hidden, NUM_LABELS],
'xavier',
xavier_params=(num_hidden, NUM_LABELS))
b_out = init_weights([1,NUM_LABELS],'zeros')
# The output layer.
y = tf.nn.softmax(tf.matmul(hidden, w_out) + b_out)
# Optimization.
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# Evaluation.
predicted_class = tf.argmax(y,1);
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Create a local session to run this computation.
with tf.Session() as s:
# Run all the initializers to prepare the trainable parameters.
tf.initialize_all_variables().run()
if verbose:
print 'Initialized!'
print
print 'Training.'
# Iterate and train.
for step in xrange(num_epochs * train_size // BATCH_SIZE):
if verbose:
print step,
offset = (step * BATCH_SIZE) % train_size
batch_data = train_data[offset:(offset + BATCH_SIZE), :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
train_step.run(feed_dict={x: batch_data, y_: batch_labels})
if verbose and offset >= train_size-BATCH_SIZE:
print
print "Accuracy:", accuracy.eval(feed_dict={x: test_data, y_: test_labels})
if plot:
eval_fun = lambda X: predicted_class.eval(feed_dict={x:X});
plot_boundary_on_data.plot(test_data, test_labels, eval_fun)
def main(argv=None):
# Be verbose?
verbose = FLAGS.verbose
# Plot?
plot = FLAGS.plot
# Get the data.
train_data_filename = FLAGS.train
test_data_filename = FLAGS.test
# Extract it into numpy matrices.
train_data,train_labels = extract_data(train_data_filename)
test_data, test_labels = extract_data(test_data_filename)
# Get the shape of the training data.
train_size,num_features = train_data.shape
# Get the number of epochs for training.
num_epochs = FLAGS.num_epochs
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
x = tf.placeholder("float", shape=[None, num_features])
y_ = tf.placeholder("float", shape=[None, NUM_LABELS])
# These are the weights that inform how much each feature contributes to
# the classification.
W = tf.Variable(tf.zeros([num_features,NUM_LABELS]))
b = tf.Variable(tf.zeros([NUM_LABELS]))
y = tf.nn.softmax(tf.matmul(x,W) + b)
# Optimization.
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# For the test data, hold the entire dataset in one constant node.
test_data_node = tf.constant(test_data)
# Evaluation.
predicted_class = tf.argmax(y,1);
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Create a local session to run this computation.
with tf.Session() as s:
# Run all the initializers to prepare the trainable parameters.
tf.initialize_all_variables().run()
# Iterate and train.
for step in xrange(num_epochs * train_size // BATCH_SIZE)
offset = (step * BATCH_SIZE) % train_size
# get a batch of data
batch_data = train_data[offset:(offset + BATCH_SIZE), :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
# feed data into the model
train_step.run(feed_dict={x: batch_data, y_: batch_labels})
# Give very detailed output.
if verbose:
print
print 'Weight matrix.'
print s.run(W)
print
print 'Bias vector.'
print s.run(b)
print
print "Applying model to first test instance."
first = test_data[:1]
print "Point =", first
print "Wx+b = ", s.run(tf.matmul(first,W)+b)
print "softmax(Wx+b) = ", s.run(tf.nn.softmax(tf.matmul(first,W)+b))
print
print "Accuracy:", accuracy.eval(feed_dict={x: test_data, y_: test_labels})
if plot:
eval_fun = lambda X: predicted_class.eval(feed_dict={x:X});
plot_boundary_on_data.plot(test_data, test_labels, eval_fun)
def main(argv=None):
# Be verbose?
verbose = FLAGS.verbose
# Plot?
plot = FLAGS.plot
# Get the data.
train_data_filename = FLAGS.train
# Extract it into numpy matrices.
train_data, train_labels = extract_data(train_data_filename)
# Convert labels to +1,-1
train_labels[train_labels == 0] = -1
# Get the shape of the training data.
train_size, num_features = train_data.shape
# Get the number of epochs for training.
num_epochs = FLAGS.num_epochs
# Get the C param of SVM
svmC = FLAGS.svmC
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
x = tf.placeholder("float", shape=[None, num_features])
y = tf.placeholder("float", shape=[None, 1])
# Define and initialize the network.
# These are the weights that inform how much each feature contributes to
# the classification.
W = tf.Variable(tf.zeros([num_features, 1]))
b = tf.Variable(tf.zeros([1]))
y_raw = tf.matmul(x, W) + b
# Optimization.
regularization_loss = 0.5 * tf.reduce_sum(tf.square(W))
hinge_loss = tf.reduce_sum(
tf.maximum(tf.zeros([BATCH_SIZE, 1]), 1 - y * y_raw))
svm_loss = regularization_loss + svmC * hinge_loss
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss)
# Evaluation.
predicted_class = tf.sign(y_raw)
correct_prediction = tf.equal(y, predicted_class)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Create a local session to run this computation.
with tf.Session() as s:
# Run all the initializers to prepare the trainable parameters.
tf.initialize_all_variables().run()
if verbose:
print 'Initialized!'
print
print 'Training.'
# Iterate and train.
for step in xrange(num_epochs * train_size // BATCH_SIZE):
if verbose:
print step,
offset = (step * BATCH_SIZE) % train_size
batch_data = train_data[offset:(offset + BATCH_SIZE), :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
train_step.run(feed_dict={x: batch_data, y: batch_labels})
print 'loss: ', svm_loss.eval(feed_dict={
x: batch_data,
y: batch_labels
})
if verbose and offset >= train_size - BATCH_SIZE:
print
# Give very detailed output.
if verbose:
print
print 'Weight matrix.'
print s.run(W)
print
print 'Bias vector.'
print s.run(b)
print
print "Applying model to first test instance."
print
print "Accuracy on train:", accuracy.eval(feed_dict={
x: train_data,
y: train_labels
})
if plot:
eval_fun = lambda X: predicted_class.eval(feed_dict={x: X})
plot_boundary_on_data.plot(train_data, train_labels, eval_fun)
data = np.load(sys.argv[1])
X = data['X']
Y = data['Y']
K = np.unique(Y).size
# define a 2-layer softmax network. First layer is a fully connected layer with
# K output nodes. The second layer is a "softmax + cross-entropy" layer.
network = [
[nn.fc, [np.random.rand(X.shape[1],K), np.random.rand(1,K)]], # first layer
[nn.softmax_cross_entropy, Y] # second layer
]
# train it
network, loss_vals = nn.train(network, X, Y, .1, 1000)
# training finished, now replace the last layer with softmax without the
# cross-entropy loss
network[-1][0] = nn.softmax_readout
predicted_class = nn.evaluate(network, X)
print('training accuracy: %.2f' % (np.mean(predicted_class == Y)))
plot_boundary_on_data.plot(X, Y, lambda x: nn.evaluate(network, x))
fig = plt.figure()
plt.plot(loss_vals)
plt.show()
def main(argv=None):
# Be verbose?
verbose = FLAGS.verbose
# Plot?
plot = FLAGS.plot
# Get the data.
train_data_filename = FLAGS.train
# Extract it into numpy matrices.
train_data,train_labels = extract_data(train_data_filename)
# Convert labels to +1,-1
train_labels[train_labels==0] = -1
# Get the shape of the training data.
train_size,num_features = train_data.shape
# Get the number of epochs for training.
num_epochs = FLAGS.num_epochs
# Get the C param of SVM
svmC = FLAGS.svmC
# This is where training samples and labels are fed to the graph.
# These placeholder nodes will be fed a batch of training data at each
# training step using the {feed_dict} argument to the Run() call below.
x = tf.placeholder("float", shape=[None, num_features])
y = tf.placeholder("float", shape=[None,1])
# Define and initialize the network.
# These are the weights that inform how much each feature contributes to
# the classification.
W = tf.Variable(tf.zeros([num_features,1]))
b = tf.Variable(tf.zeros([1]))
y_raw = tf.matmul(x,W) + b
# Optimization.
regularization_loss = 0.5*tf.reduce_sum(tf.square(W))
hinge_loss = tf.reduce_sum(tf.maximum(tf.zeros([BATCH_SIZE,1]),
1 - y*y_raw));
svm_loss = regularization_loss + svmC*hinge_loss;
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss)
# Evaluation.
predicted_class = tf.sign(y_raw);
correct_prediction = tf.equal(y,predicted_class)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Create a local session to run this computation.
with tf.Session() as s:
# Run all the initializers to prepare the trainable parameters.
tf.initialize_all_variables().run()
if verbose:
print 'Initialized!'
print
print 'Training.'
# Iterate and train.
for step in xrange(num_epochs * train_size // BATCH_SIZE):
if verbose:
print step,
offset = (step * BATCH_SIZE) % train_size
batch_data = train_data[offset:(offset + BATCH_SIZE), :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
train_step.run(feed_dict={x: batch_data, y: batch_labels})
print 'loss: ', svm_loss.eval(feed_dict={x: batch_data, y: batch_labels})
if verbose and offset >= train_size-BATCH_SIZE:
print
# Give very detailed output.
if verbose:
print
print 'Weight matrix.'
print s.run(W)
print
print 'Bias vector.'
print s.run(b)
print
print "Applying model to first test instance."
print
print "Accuracy on train:", accuracy.eval(feed_dict={x: train_data, y: train_labels})
if plot:
eval_fun = lambda X: predicted_class.eval(feed_dict={x:X});
plot_boundary_on_data.plot(train_data, train_labels, eval_fun)
def main(argv=None):
# Be verbose?
verbose = FLAGS.verbose
# Plot?
plot = FLAGS.plot
# Get the data.
train_data_filename = FLAGS.train
test_data_filename = FLAGS.test
# load data
train_data,train_labels = extract_data(train_data_filename)
test_data, test_labels = extract_data(test_data_filename)
# Get the shape of the training data.
train_size,num_features = train_data.shape
# Get the number of epochs for training.
num_epochs = FLAGS.num_epochs
# Get the size of layer one.
num_hidden = FLAGS.num_hidden
# This is where training samples and labels are fed to the graph.
with tf.name_scope('inputLayers'):
x = tf.placeholder("float", shape=[None, num_features], name='x_input')
y_ = tf.placeholder("float", shape=[None, NUM_LABELS], name= 'y_input')
# For the test data, hold the entire dataset in one constant node.
test_data_node = tf.constant(test_data)
# The hidden layer.
with tf.name_scope('hiddenLayer'):
w_hidden = init_weights(
[num_features, num_hidden],'xavier',
xavier_params=(num_features, num_hidden))
b_hidden = init_weights([1,num_hidden],'zeros')
hidden = tf.nn.tanh(tf.matmul(x,w_hidden) + b_hidden)
# The output layer.
with tf.name_scope('outputLayer'):
# Initialize the output weights and biases.
w_out = init_weights(
[num_hidden, NUM_LABELS],'xavier',
xavier_params=(num_hidden, NUM_LABELS))
b_out = init_weights([1,NUM_LABELS],'zeros')
y = tf.nn.softmax(tf.matmul(hidden, w_out) + b_out)
# Optimization.
with tf.name_scope('loss'):
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_*tf.log(y)))
tf.summary.scalar('loss', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
# Evaluation.
predicted_class = tf.argmax(y,1);
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
# Create a local session to run this computation.
with tf.Session() as sess:
# Run all the initializers to prepare the trainable parameters.
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter("logs/train", sess.graph)
test_writer = tf.summary.FileWriter("logs/test", sess.graph)
if verbose:
print ('Initialized!')
print ('Training.')
# Iterate and train.
Iteratation_num = num_epochs * train_size // BATCH_SIZE
print(Iteratation_num)
# initail all and run
tf.global_variables_initializer().run()
for step in range(num_epochs * train_size // BATCH_SIZE):
if verbose:
print (step),
offset = (step * BATCH_SIZE) % train_size
batch_data = train_data[offset:(offset + BATCH_SIZE), :]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
_,loss = sess.run([train_step, cross_entropy], feed_dict={x: batch_data, y_: batch_labels})
if verbose and offset >= train_size-BATCH_SIZE:
print('verbose and offset >= train_size-BATCH_SIZE')
print('step %d: loss = %.2f' %(step, loss))
if step % 50 == 0:
# record loss
train_result = sess.run(merged, feed_dict={x: batch_data, y_: batch_labels})
test_result = sess.run(merged, feed_dict={x: test_data, y_: test_labels})
train_writer.add_summary(train_result, step)
test_writer.add_summary(test_result, step)
print ("Accuracy is :", accuracy.eval(feed_dict={x: test_data, y_: test_labels}))
if plot:
eval_fun = lambda X: predicted_class.eval(feed_dict={x:X});
plot_boundary_on_data.plot(test_data, test_labels, eval_fun)
