# Read in training data with gzip.open(sys.argv[1]) as f: data, labels = pickle.load(f) # Starting values for w and b w0 = np.zeros(data.shape[1]) b0 = 0 # Optimization w, b = hw1.minimize_l2loss(data, labels, w0, b0, int(sys.argv[3]), float(sys.argv[4])) # Show the result of training #print w,b if w.size == 2: plot_boundary_on_data.plot(data, labels, lambda x: hw1.f(x, w, b) > .5) if w.size == 784: # special to MNIST plt.imshow(w.reshape(28, 28)) plt.show() # Test on test data with gzip.open(sys.argv[2]) as f: test_data, test_labels = pickle.load(f) yhat = hw1.f(test_data, w, b) >= .5 print np.mean( yhat == test_labels) * 100, "% of test examples classified correctly." hw1.evaluate(hw1.f(test_data, w, b), test_labels)
def main(argv=None): verbose = FLAGS.verbose plot = FLAGS.plot train_data_filename = FLAGS.train train_data,train_labels = extract_data(train_data_filename) train_labels[train_labels==0] = -1 train_size,num_features = train_data.shape num_epochs = FLAGS.num_epochs svmC = FLAGS.svmC x = tf.placeholder("float", shape=[None, num_features]) y = tf.placeholder("float", shape=[None,1]) W = tf.Variable(tf.zeros([num_features,1])) b = tf.Variable(tf.zeros([1])) y_raw = tf.matmul(x,W) + b # Optimization. regularization_loss = 0.5*tf.reduce_sum(tf.square(W)) hinge_loss = tf.reduce_sum(tf.maximum(tf.zeros([BATCH_SIZE,1]), 1 - y*y_raw)); svm_loss = regularization_loss + svmC*hinge_loss; train_step = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss) # Evaluation. predicted_class = tf.sign(y_raw); correct_prediction = tf.equal(y,predicted_class) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) with tf.Session() as s: tf.initialize_all_variables().run() if verbose: print ('Initialized!') print () print ('Training.') for step in xrange(num_epochs * train_size // BATCH_SIZE): if verbose: print (step), offset = (step * BATCH_SIZE) % train_size batch_data = train_data[offset:(offset + BATCH_SIZE), :] batch_labels = train_labels[offset:(offset + BATCH_SIZE)] train_step.run(feed_dict={x: batch_data, y: batch_labels}) print ('loss: ', svm_loss.eval(feed_dict={x: batch_data, y: batch_labels})) if verbose and offset >= train_size-BATCH_SIZE: print if verbose: print print ('Weight matrix.') print (s.run(W)) print print ('Bias vector.') print (s.run(b)) print print ("Applying model to first test instance.") print print ("Accuracy on train:", accuracy.eval(feed_dict={x: train_data, y: train_labels})) if plot: eval_fun = lambda X: predicted_class.eval(feed_dict={x:X}); plot_boundary_on_data.plot(train_data, train_labels, eval_fun)
def main(argv=None): # Be verbose? verbose = FLAGS.verbose # Plot? plot = FLAGS.plot # Get the data. train_data_filename = FLAGS.train test_data_filename = FLAGS.test # Extract it into numpy arrays. train_data,train_labels = extract_data(train_data_filename) test_data, test_labels = extract_data(test_data_filename) # Get the shape of the training data. train_size,num_features = train_data.shape # Get the number of epochs for training. num_epochs = FLAGS.num_epochs # Get the size of layer one. num_hidden = FLAGS.num_hidden # This is where training samples and labels are fed to the graph. # These placeholder nodes will be fed a batch of training data at each # training step using the {feed_dict} argument to the Run() call below. x = tf.placeholder("float", shape=[None, num_features]) y_ = tf.placeholder("float", shape=[None, NUM_LABELS]) # For the test data, hold the entire dataset in one constant node. test_data_node = tf.constant(test_data) # Define and initialize the network. # Initialize the hidden weights and biases. w_hidden = init_weights( [num_features, num_hidden], 'xavier', xavier_params=(num_features, num_hidden)) b_hidden = init_weights([1,num_hidden],'zeros') # The hidden layer. hidden = tf.nn.tanh(tf.matmul(x,w_hidden) + b_hidden) # Initialize the output weights and biases. w_out = init_weights( [num_hidden, NUM_LABELS], 'xavier', xavier_params=(num_hidden, NUM_LABELS)) b_out = init_weights([1,NUM_LABELS],'zeros') # The output layer. y = tf.nn.softmax(tf.matmul(hidden, w_out) + b_out) # Optimization. cross_entropy = -tf.reduce_sum(y_*tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) # Evaluation. predicted_class = tf.argmax(y,1); correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # Create a local session to run this computation. with tf.Session() as s: # Run all the initializers to prepare the trainable parameters. tf.initialize_all_variables().run() if verbose: print 'Initialized!' print print 'Training.' # Iterate and train. for step in xrange(num_epochs * train_size // BATCH_SIZE): if verbose: print step, offset = (step * BATCH_SIZE) % train_size batch_data = train_data[offset:(offset + BATCH_SIZE), :] batch_labels = train_labels[offset:(offset + BATCH_SIZE)] train_step.run(feed_dict={x: batch_data, y_: batch_labels}) if verbose and offset >= train_size-BATCH_SIZE: print print "Accuracy:", accuracy.eval(feed_dict={x: test_data, y_: test_labels}) if plot: eval_fun = lambda X: predicted_class.eval(feed_dict={x:X}); plot_boundary_on_data.plot(test_data, test_labels, eval_fun)
def main(argv=None): # Be verbose? verbose = FLAGS.verbose # Plot? plot = FLAGS.plot # Get the data. train_data_filename = FLAGS.train test_data_filename = FLAGS.test # Extract it into numpy matrices. train_data,train_labels = extract_data(train_data_filename) test_data, test_labels = extract_data(test_data_filename) # Get the shape of the training data. train_size,num_features = train_data.shape # Get the number of epochs for training. num_epochs = FLAGS.num_epochs # This is where training samples and labels are fed to the graph. # These placeholder nodes will be fed a batch of training data at each # training step using the {feed_dict} argument to the Run() call below. x = tf.placeholder("float", shape=[None, num_features]) y_ = tf.placeholder("float", shape=[None, NUM_LABELS]) # These are the weights that inform how much each feature contributes to # the classification. W = tf.Variable(tf.zeros([num_features,NUM_LABELS])) b = tf.Variable(tf.zeros([NUM_LABELS])) y = tf.nn.softmax(tf.matmul(x,W) + b) # Optimization. cross_entropy = -tf.reduce_sum(y_*tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) # For the test data, hold the entire dataset in one constant node. test_data_node = tf.constant(test_data) # Evaluation. predicted_class = tf.argmax(y,1); correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # Create a local session to run this computation. with tf.Session() as s: # Run all the initializers to prepare the trainable parameters. tf.initialize_all_variables().run() # Iterate and train. for step in xrange(num_epochs * train_size // BATCH_SIZE) offset = (step * BATCH_SIZE) % train_size # get a batch of data batch_data = train_data[offset:(offset + BATCH_SIZE), :] batch_labels = train_labels[offset:(offset + BATCH_SIZE)] # feed data into the model train_step.run(feed_dict={x: batch_data, y_: batch_labels}) # Give very detailed output. if verbose: print print 'Weight matrix.' print s.run(W) print print 'Bias vector.' print s.run(b) print print "Applying model to first test instance." first = test_data[:1] print "Point =", first print "Wx+b = ", s.run(tf.matmul(first,W)+b) print "softmax(Wx+b) = ", s.run(tf.nn.softmax(tf.matmul(first,W)+b)) print print "Accuracy:", accuracy.eval(feed_dict={x: test_data, y_: test_labels}) if plot: eval_fun = lambda X: predicted_class.eval(feed_dict={x:X}); plot_boundary_on_data.plot(test_data, test_labels, eval_fun)
def main(argv=None): # Be verbose? verbose = FLAGS.verbose # Plot? plot = FLAGS.plot # Get the data. train_data_filename = FLAGS.train # Extract it into numpy matrices. train_data, train_labels = extract_data(train_data_filename) # Convert labels to +1,-1 train_labels[train_labels == 0] = -1 # Get the shape of the training data. train_size, num_features = train_data.shape # Get the number of epochs for training. num_epochs = FLAGS.num_epochs # Get the C param of SVM svmC = FLAGS.svmC # This is where training samples and labels are fed to the graph. # These placeholder nodes will be fed a batch of training data at each # training step using the {feed_dict} argument to the Run() call below. x = tf.placeholder("float", shape=[None, num_features]) y = tf.placeholder("float", shape=[None, 1]) # Define and initialize the network. # These are the weights that inform how much each feature contributes to # the classification. W = tf.Variable(tf.zeros([num_features, 1])) b = tf.Variable(tf.zeros([1])) y_raw = tf.matmul(x, W) + b # Optimization. regularization_loss = 0.5 * tf.reduce_sum(tf.square(W)) hinge_loss = tf.reduce_sum( tf.maximum(tf.zeros([BATCH_SIZE, 1]), 1 - y * y_raw)) svm_loss = regularization_loss + svmC * hinge_loss train_step = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss) # Evaluation. predicted_class = tf.sign(y_raw) correct_prediction = tf.equal(y, predicted_class) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # Create a local session to run this computation. with tf.Session() as s: # Run all the initializers to prepare the trainable parameters. tf.initialize_all_variables().run() if verbose: print 'Initialized!' print print 'Training.' # Iterate and train. for step in xrange(num_epochs * train_size // BATCH_SIZE): if verbose: print step, offset = (step * BATCH_SIZE) % train_size batch_data = train_data[offset:(offset + BATCH_SIZE), :] batch_labels = train_labels[offset:(offset + BATCH_SIZE)] train_step.run(feed_dict={x: batch_data, y: batch_labels}) print 'loss: ', svm_loss.eval(feed_dict={ x: batch_data, y: batch_labels }) if verbose and offset >= train_size - BATCH_SIZE: print # Give very detailed output. if verbose: print print 'Weight matrix.' print s.run(W) print print 'Bias vector.' print s.run(b) print print "Applying model to first test instance." print print "Accuracy on train:", accuracy.eval(feed_dict={ x: train_data, y: train_labels }) if plot: eval_fun = lambda X: predicted_class.eval(feed_dict={x: X}) plot_boundary_on_data.plot(train_data, train_labels, eval_fun)
data = np.load(sys.argv[1]) X = data['X'] Y = data['Y'] K = np.unique(Y).size # define a 2-layer softmax network. First layer is a fully connected layer with # K output nodes. The second layer is a "softmax + cross-entropy" layer. network = [ [nn.fc, [np.random.rand(X.shape[1],K), np.random.rand(1,K)]], # first layer [nn.softmax_cross_entropy, Y] # second layer ] # train it network, loss_vals = nn.train(network, X, Y, .1, 1000) # training finished, now replace the last layer with softmax without the # cross-entropy loss network[-1][0] = nn.softmax_readout predicted_class = nn.evaluate(network, X) print('training accuracy: %.2f' % (np.mean(predicted_class == Y))) plot_boundary_on_data.plot(X, Y, lambda x: nn.evaluate(network, x)) fig = plt.figure() plt.plot(loss_vals) plt.show()
def main(argv=None): # Be verbose? verbose = FLAGS.verbose # Plot? plot = FLAGS.plot # Get the data. train_data_filename = FLAGS.train # Extract it into numpy matrices. train_data,train_labels = extract_data(train_data_filename) # Convert labels to +1,-1 train_labels[train_labels==0] = -1 # Get the shape of the training data. train_size,num_features = train_data.shape # Get the number of epochs for training. num_epochs = FLAGS.num_epochs # Get the C param of SVM svmC = FLAGS.svmC # This is where training samples and labels are fed to the graph. # These placeholder nodes will be fed a batch of training data at each # training step using the {feed_dict} argument to the Run() call below. x = tf.placeholder("float", shape=[None, num_features]) y = tf.placeholder("float", shape=[None,1]) # Define and initialize the network. # These are the weights that inform how much each feature contributes to # the classification. W = tf.Variable(tf.zeros([num_features,1])) b = tf.Variable(tf.zeros([1])) y_raw = tf.matmul(x,W) + b # Optimization. regularization_loss = 0.5*tf.reduce_sum(tf.square(W)) hinge_loss = tf.reduce_sum(tf.maximum(tf.zeros([BATCH_SIZE,1]), 1 - y*y_raw)); svm_loss = regularization_loss + svmC*hinge_loss; train_step = tf.train.GradientDescentOptimizer(0.01).minimize(svm_loss) # Evaluation. predicted_class = tf.sign(y_raw); correct_prediction = tf.equal(y,predicted_class) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # Create a local session to run this computation. with tf.Session() as s: # Run all the initializers to prepare the trainable parameters. tf.initialize_all_variables().run() if verbose: print 'Initialized!' print print 'Training.' # Iterate and train. for step in xrange(num_epochs * train_size // BATCH_SIZE): if verbose: print step, offset = (step * BATCH_SIZE) % train_size batch_data = train_data[offset:(offset + BATCH_SIZE), :] batch_labels = train_labels[offset:(offset + BATCH_SIZE)] train_step.run(feed_dict={x: batch_data, y: batch_labels}) print 'loss: ', svm_loss.eval(feed_dict={x: batch_data, y: batch_labels}) if verbose and offset >= train_size-BATCH_SIZE: print # Give very detailed output. if verbose: print print 'Weight matrix.' print s.run(W) print print 'Bias vector.' print s.run(b) print print "Applying model to first test instance." print print "Accuracy on train:", accuracy.eval(feed_dict={x: train_data, y: train_labels}) if plot: eval_fun = lambda X: predicted_class.eval(feed_dict={x:X}); plot_boundary_on_data.plot(train_data, train_labels, eval_fun)
def main(argv=None): # Be verbose? verbose = FLAGS.verbose # Plot? plot = FLAGS.plot # Get the data. train_data_filename = FLAGS.train test_data_filename = FLAGS.test # load data train_data,train_labels = extract_data(train_data_filename) test_data, test_labels = extract_data(test_data_filename) # Get the shape of the training data. train_size,num_features = train_data.shape # Get the number of epochs for training. num_epochs = FLAGS.num_epochs # Get the size of layer one. num_hidden = FLAGS.num_hidden # This is where training samples and labels are fed to the graph. with tf.name_scope('inputLayers'): x = tf.placeholder("float", shape=[None, num_features], name='x_input') y_ = tf.placeholder("float", shape=[None, NUM_LABELS], name= 'y_input') # For the test data, hold the entire dataset in one constant node. test_data_node = tf.constant(test_data) # The hidden layer. with tf.name_scope('hiddenLayer'): w_hidden = init_weights( [num_features, num_hidden],'xavier', xavier_params=(num_features, num_hidden)) b_hidden = init_weights([1,num_hidden],'zeros') hidden = tf.nn.tanh(tf.matmul(x,w_hidden) + b_hidden) # The output layer. with tf.name_scope('outputLayer'): # Initialize the output weights and biases. w_out = init_weights( [num_hidden, NUM_LABELS],'xavier', xavier_params=(num_hidden, NUM_LABELS)) b_out = init_weights([1,NUM_LABELS],'zeros') y = tf.nn.softmax(tf.matmul(hidden, w_out) + b_out) # Optimization. with tf.name_scope('loss'): cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_*tf.log(y))) tf.summary.scalar('loss', cross_entropy) with tf.name_scope('train'): train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) # Evaluation. predicted_class = tf.argmax(y,1); correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) # Create a local session to run this computation. with tf.Session() as sess: # Run all the initializers to prepare the trainable parameters. merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter("logs/train", sess.graph) test_writer = tf.summary.FileWriter("logs/test", sess.graph) if verbose: print ('Initialized!') print ('Training.') # Iterate and train. Iteratation_num = num_epochs * train_size // BATCH_SIZE print(Iteratation_num) # initail all and run tf.global_variables_initializer().run() for step in range(num_epochs * train_size // BATCH_SIZE): if verbose: print (step), offset = (step * BATCH_SIZE) % train_size batch_data = train_data[offset:(offset + BATCH_SIZE), :] batch_labels = train_labels[offset:(offset + BATCH_SIZE)] _,loss = sess.run([train_step, cross_entropy], feed_dict={x: batch_data, y_: batch_labels}) if verbose and offset >= train_size-BATCH_SIZE: print('verbose and offset >= train_size-BATCH_SIZE') print('step %d: loss = %.2f' %(step, loss)) if step % 50 == 0: # record loss train_result = sess.run(merged, feed_dict={x: batch_data, y_: batch_labels}) test_result = sess.run(merged, feed_dict={x: test_data, y_: test_labels}) train_writer.add_summary(train_result, step) test_writer.add_summary(test_result, step) print ("Accuracy is :", accuracy.eval(feed_dict={x: test_data, y_: test_labels})) if plot: eval_fun = lambda X: predicted_class.eval(feed_dict={x:X}); plot_boundary_on_data.plot(test_data, test_labels, eval_fun)