#!/usr/bin/env python import tensorflow as tf import read_inputs import numpy as N GPUS = 1 BATCH_SIZE = 50 * GPUS EPOCHS = 30 #read data from file data_input = read_inputs.load_data_mnist('MNIST_data/mnist.pkl.gz') #FYI data = [(train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y)] data = data_input[0] #print ( N.shape(data[0][0])[0] ) #print ( N.shape(data[0][1])[0] ) #data layout changes since output should an array of 10 with probabilities real_output = N.zeros((N.shape(data[0][1])[0], 10), dtype=N.float) for i in range(N.shape(data[0][1])[0]): real_output[i][data[0][1][i]] = 1.0 #data layout changes since output should an array of 10 with probabilities real_check = N.zeros((N.shape(data[2][1])[0], 10), dtype=N.float) for i in range(N.shape(data[2][1])[0]): real_check[i][data[2][1][i]] = 1.0 #set up the computation. Definition of the variables. x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(tf.zeros([784, 10])) y_ = tf.placeholder(tf.float32, [None, 10]) keep_prob = tf.placeholder(tf.float32)
def main(epochs=10, do=0.5, batch_size=50, gpu_num=1): gpu_names = check_available_gpus() #gpu_num = len(gpu_names) print() print('# of GPUs:', gpu_num) print('GPU devices:', gpu_names) print() val_batch_size = 100 #read data from file data_input = read_inputs.load_data_mnist('MNIST_data/mnist.pkl.gz') #FYI data = [(train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y)] data = data_input[0] batch_num = len(data[0][0]) // (batch_size * gpu_num) val_batch_num = len(data[1][0]) // (val_batch_size * gpu_num) new_tr_ind = random.sample(range(len(data[0][0])), len(data[0][0])) new_val_ind = random.sample(range(len(data[1][0])), len(data[1][0])) #print ( N.shape(data[0][0])[0] ) #print ( N.shape(data[0][1])[0] ) #data layout changes since output should an array of 10 with probabilities real_output = N.zeros((N.shape(data[0][1])[0], 10), dtype=N.float) for i in range(N.shape(data[0][1])[0]): real_output[i][data[0][1][i]] = 1.0 val_output = N.zeros((N.shape(data[1][1])[0], 10), dtype=N.float) for i in range(N.shape(data[1][1])[0]): val_output[i][data[1][1][i]] = 1.0 #data layout changes since output should an array of 10 with probabilities real_check = N.zeros((N.shape(data[2][1])[0], 10), dtype=N.float) for i in range(N.shape(data[2][1])[0]): real_check[i][data[2][1][i]] = 1.0 new_X_data = data[0][0][new_tr_ind] new_y_data = real_output[new_tr_ind] new_valX_data = data[1][0][new_val_ind] new_valy_data = val_output[new_val_ind] # Place all ops on CPU by default with tf.device('/cpu:0'): tower_losses = [] reuse_vars = False # tf Graph input #set up the computation. Definition of the variables. x = tf.placeholder(tf.float32, [None, 784]) #W = tf.Variable(tf.zeros([784, 10])) y_ = tf.placeholder(tf.float32, [None, 10]) keep_prob = tf.placeholder(tf.float32) # Loop over all GPUs and construct their own computation graph for i in range(gpu_num): with tf.device('/gpu:{}'.format(i)): _batch_ini = batch_size * i _batch_end = batch_size * i + batch_size _batch_xs = x[_batch_ini:_batch_end] _batch_ys = y_[_batch_ini:_batch_end] y_conv = get_model(reuse_vars, _batch_xs, True, keep_prob) print() print(_batch_ys.get_shape(), y_conv.get_shape()) #Crossentropy # Define loss and optimizer (with train logits, for dropout to take effect) #cross_entropy = tf.reduce_mean( # tf.nn.softmax_cross_entropy_with_logits(labels=_batch_ys, logits=y_conv)) cross_entropy = tf.nn.softmax_cross_entropy_with_logits( labels=_batch_ys, logits=y_conv) # Only first GPU compute accuracy if i == 0: correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(_batch_ys, 1)) accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) reuse_vars = True tower_losses.append(cross_entropy) loss = tf.reduce_mean(tf.concat(tower_losses, axis=0)) train_step = tf.train.AdamOptimizer(0.001).minimize( loss, colocate_gradients_with_ops=True) print('Exp:', batch_size, do) with tf.Session(config=tf.ConfigProto(log_device_placement=False)) as sess: sess.run(tf.global_variables_initializer()) #TRAIN print("TRAINING") loss_lst = [] val_loss_lst = [] tr_acc_lst = [] val_acc_lst = [] start = time() for epoch in range(epochs): loss_batch = 0 acc_batch = 0 for i in range(batch_num): #until 1000 96,35% batch_ini = batch_size * i * gpu_num batch_end = batch_size * i * gpu_num + batch_size * gpu_num batch_xs = new_X_data[batch_ini:batch_end] #data[0][0][batch_ini:batch_end] batch_ys = new_y_data[batch_ini:batch_end] #real_output[batch_ini:batch_end] ''' if i % 10 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batch_xs, y_: batch_ys, keep_prob: 1}) print('step %d, training accuracy %g Batch [%d,%d]' % (i, train_accuracy, batch_ini, batch_end)) ''' sess.run(train_step, feed_dict={ x: batch_xs, y_: batch_ys, keep_prob: do }) train_accuracy = accuracy.eval(feed_dict={ x: batch_xs, y_: batch_ys, keep_prob: 1 }) curr_loss = sess.run(loss, { x: batch_xs, y_: batch_ys, keep_prob: do }) loss_batch += curr_loss acc_batch += train_accuracy loss_lst.append(loss_batch / batch_num) tr_acc_lst.append(acc_batch / batch_num) loss_batch = 0 acc_batch = 0 for i in range(val_batch_num): val_batch_ini = val_batch_size * i * gpu_num val_batch_end = val_batch_size * i * gpu_num + val_batch_size * gpu_num batch_val_xs = new_valX_data[val_batch_ini:val_batch_end] batch_val_ys = new_valy_data[val_batch_ini:val_batch_end] val_loss = sess.run(loss, { x: batch_val_xs, y_: batch_val_ys, keep_prob: do }) #val_acc = accuracy.eval(feed_dict={x: batch_val_xs, y_: batch_val_ys, keep_prob: 1}) val_acc = sess.run(accuracy, feed_dict={ x: batch_val_xs, y_: batch_val_ys, keep_prob: 1 }) loss_batch += val_loss acc_batch += val_acc val_loss_lst.append(loss_batch / val_batch_num) val_acc_lst.append(acc_batch / val_batch_num) percent = 100.0 * epoch / epochs line = '[{0}{1}]'.format('=' * int(percent / 2), ' ' * (50 - int(percent / 2))) status = '\r{0:3.0f}%{1} {2:3d}/{3:3d} loss:{4:.2f} val_loss:{5:.2f} acc:{6:.2f} val_acc:{7:.2f}' sys.stdout.write( status.format(percent, line, epoch, epochs, loss_lst[-1], val_loss_lst[-1], tr_acc_lst[-1], val_acc_lst[-1])) duration = time() - start print() print('Training duration:', duration) print() #TEST print("TESTING") #tf_acc = accuracy.eval(feed_dict={x: data[2][0], y_: real_check, keep_prob: 1}) tf_acc = N.mean([ sess.run(accuracy, feed_dict={ x: data[2][0][i:i + batch_size], y_: real_check[i:i + batch_size], keep_prob: 1 }) for i in range(0, len(data[2][0]), batch_size) ]) print('test accuracy %g' % (tf_acc)) return N.array(loss_lst), N.array(val_loss_lst), N.array( tr_acc_lst), N.array(val_acc_lst), duration, tf_acc
def run_train(step): #read data from file data_input = read_inputs.load_data_mnist('MNIST_data/mnist.pkl.gz') #FYI data = [(train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y)] data = data_input[0] #print ( N.shape(data[0][0])[0] ) #print ( N.shape(data[0][1])[0] ) #data layout changes since output should an array of 10 with probabilities real_output = N.zeros((N.shape(data[0][1])[0], 10), dtype=N.float) for i in range(N.shape(data[0][1])[0]): real_output[i][data[0][1][i]] = 1.0 #data layout changes since output should an array of 10 with probabilities real_check = N.zeros((N.shape(data[2][1])[0], 10), dtype=N.float) for i in range(N.shape(data[2][1])[0]): real_check[i][data[2][1][i]] = 1.0 #set up the computation. Definition of the variables. x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, 10]) cross_entropy = tf.reduce_mean( -tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) train_step = tf.train.AdagradOptimizer(step).minimize(cross_entropy) sess = tf.InteractiveSession() tf.global_variables_initializer().run() #TRAINING PHASE print("TRAINING") accs = [] losses = [] for i in range(500): batch_xs = data[0][0][100 * i:100 * i + 100] batch_ys = real_output[100 * i:100 * i + 100] sess.run([train_step, cross_entropy], feed_dict={ x: batch_xs, y_: batch_ys }) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) acc, loss = sess.run([accuracy, cross_entropy], feed_dict={ x: data[2][0], y_: real_check }) accs.append(acc) losses.append(loss) #CHECKING THE ERROR print("ERROR CHECK") correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) print(sess.run(accuracy, feed_dict={x: data[2][0], y_: real_check})) out = 1 return accs, losses
def main(optimizer, epochs=5): #read data from file data_input = read_inputs.load_data_mnist('MNIST_data/mnist.pkl.gz') #FYI data = [(train_set_x, train_set_y), (valid_set_x, valid_set_y), (test_set_x, test_set_y)] data = data_input[0] #print ( N.shape(data[0][0])[0] ) #print ( N.shape(data[0][1])[0] ) print(N.array(data)[0][0].shape) print(N.array(data)[1][0].shape) print(N.array(data)[2][0].shape) #data layout changes since output should an array of 10 with probabilities real_output = N.zeros((N.shape(data[0][1])[0], 10), dtype=N.float) for i in range(N.shape(data[0][1])[0]): real_output[i][data[0][1][i]] = 1.0 val_output = N.zeros((N.shape(data[1][1])[0], 10), dtype=N.float) for i in range(N.shape(data[1][1])[0]): val_output[i][data[1][1][i]] = 1.0 #data layout changes since output should be an array of 10 with probabilities real_check = N.zeros((N.shape(data[2][1])[0], 10), dtype=N.float) for i in range(N.shape(data[2][1])[0]): real_check[i][data[2][1][i]] = 1.0 #set up the computation. Definition of the variables. x = tf.placeholder(tf.float32, [None, 784]) W = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, 10]) cross_entropy = tf.reduce_mean( -tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) # tf.train.GradientDescentOptimizer(0.5) train_step = optimizer.minimize(cross_entropy) #sess = tf.InteractiveSession() #TRAINING PHASE print("TRAINING") with tf.Session() as sess: tf.global_variables_initializer().run() correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) loss_lst = [] val_loss_lst = [] tr_acc_lst = [] val_acc_lst = [] start = time() for epoch in range(epochs): loss_batch = 0 acc_batch = 0 for i in range(500): batch_xs = data[0][0][100 * i:100 * i + 100] batch_ys = real_output[100 * i:100 * i + 100] sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) curr_loss = sess.run(cross_entropy, { x: batch_xs, y_: batch_ys }) tr_acc = sess.run(accuracy, feed_dict={ x: batch_xs, y_: batch_ys }) loss_batch += curr_loss acc_batch += tr_acc loss_lst.append(loss_batch / 500) tr_acc_lst.append(acc_batch / 500) loss_batch = 0 acc_batch = 0 for i in range(100): batch_val_xs = data[1][0][100 * i:100 * i + 100] batch_val_ys = val_output[100 * i:100 * i + 100] val_loss = sess.run(cross_entropy, { x: batch_val_xs, y_: batch_val_ys }) val_acc = sess.run(accuracy, feed_dict={ x: batch_val_xs, y_: batch_val_ys }) loss_batch += val_loss acc_batch += val_acc val_loss_lst.append(loss_batch / 100) val_acc_lst.append(acc_batch / 100) percent = 100.0 * epoch / epochs line = '[{0}{1}]'.format('=' * int(percent / 2), ' ' * (50 - int(percent / 2))) status = '\r{0:3.0f}%{1} {2:3d}/{3:3d}' sys.stdout.write(status.format(percent, line, epoch, epochs)) sys.stdout.flush() duration = time() - start #CHECKING THE ERROR print("ERROR CHECK") #correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) #accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf_acc = accuracy.eval(feed_dict={x: data[2][0], y_: real_check}) return N.array(loss_lst), N.array(val_loss_lst), N.array( tr_acc_lst), N.array(val_acc_lst), duration, tf_acc