def runNet(datafile=N4_DATA_FILE, tensorboard_dir=N4_TENSORBOARD_DIR, save_dir=N4_SAVE_DIR, model_id=N4_MODEL_ID, image_dir=N4_IMG_DIR, load_model=N4_LOAD_MODEL, save_model=N4_SAVE_MODEL, write_image=N4_WRITE_IMAGES, batch_size=N4_BATCH_SIZE, total_steps=N4_TOTAL_STEPS, learning_rate=N4_LEARNING_RATE, other_params={}): # Load and separate datasets data = h5py.File(datafile) train_dataset = np.transpose(data.get('train_inputs')) train_labels = np.transpose(data.get('train_labels')) valid_dataset = np.transpose(data.get('valid_inputs')) valid_labels = np.transpose(data.get('valid_labels')) test_dataset = np.transpose(data.get('test_inputs')) test_labels = np.transpose(data.get('test_labels')) activation = other_params.get('activation', N4_ACTIVATION) #Get input size kx = int(data.get('kx')[0]) ky = int(data.get('ky')[0]) # NETWORK PARAMS num_input_units = kx * ky #HIDDEN_UNITS = 16 num_output_units = kx * ky print("INPUT_UNITS:", num_input_units, "OUTPUT_UNITS:", num_output_units) # Build graph # Using the standard graph with tf.Graph().as_default(): # Network inputs and outputs input_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_input_units)) label_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_output_units)) # Define output layer with tf.name_scope('out_layer'): weights = tf.Variable(tf.truncated_normal( [num_input_units, num_output_units], stddev=0.1), name='weights') biases = tf.Variable(tf.truncated_normal([num_output_units], stddev=0.1), name='biases') h_logits = tf.matmul(input_placeholder, weights) + biases # Hidden layer activations? if activation == 'relu': logits = tf.nn.relu(h_logits, name='h_output') elif activation == 'sigmoid': logits = tf.sigmoid(h_logits, name='h_output') elif activation == 'linear': logits = h_logits else: raise Exception('Activation not known: ' + activation) tf.histogram_summary("out_biases", biases) tf.histogram_summary("out_weights", weights) tf.histogram_summary("logits", logits) # loss function - Sum squared difference (well mean...) loss = tf.reduce_mean(tf.square(label_placeholder - logits), name='loss') # Log data tf.scalar_summary(loss.op.name, loss) # Training operations optimiser = tf.train.GradientDescentOptimizer(learning_rate) global_step = tf.Variable(0, name='global_step', trainable=False) train_op = optimiser.minimize(loss, global_step=global_step) # Collect all summaries together into one operation summary_op = tf.merge_all_summaries() # Saver will save the state of the network in case of crash etc. saver = tf.train.Saver() # Run graph sess = tf.Session() # Initialise variables if not load_model: init = tf.initialize_all_variables() sess.run(init) else: saver.restore(sess, save_dir + model_id) print("Restored network from file: %s" % save_dir + model_id) summary_writer = tf.train.SummaryWriter(tensorboard_dir, graph_def=sess.graph_def) for step in xrange(total_steps): # Create feed dictionary offset = (step * batch_size) % (train_labels.shape[0] - batch_size) batch_data = train_dataset[offset:(offset + batch_size), :] batch_labels = train_labels[offset:(offset + batch_size), :] feed_dict = { input_placeholder: batch_data, label_placeholder: batch_labels } _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) if step % 250 == 0: summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) # Save a checkpoint and evaluate the model periodically. if step % total_steps - 1 == 0: eval_model(step, batch_size, valid_labels, valid_dataset, sess, logits, loss, input_placeholder, label_placeholder) """ if step % total_steps - 1 == 0: # RUN Network with validation data offset = (step * batch_size) % (valid_labels.shape[0] - batch_size) batch_data = valid_dataset[offset : (offset + batch_size), :] batch_labels = valid_labels[offset : (offset + batch_size), :] feed_dict = { input_placeholder : batch_data, label_placeholder : batch_labels } preds, validation_value, = sess.run([logits, loss], feed_dict=feed_dict) print("Step: %d: validation: %.5f" % (step, validation_value)) """ if save_model: save_path = saver.save(sess, save_dir + model_id) print("Model saved in file: %s" % save_path) ## If on my laptop then just write straight to images if write_image: print("Attempting to write images now...") import visTools batch_data, batch_labels, preds = eval_model( step, batch_size, valid_labels, valid_dataset, sess, logits, loss, input_placeholder, label_placeholder) visTools.write_preds(batch_data, batch_labels, preds, image_dir, kx)
def runNet(datafile=N6_DATA_FILE, tensorboard_dir=N6_TENSORBOARD_DIR, save_dir=N6_SAVE_DIR, model_id=N6_MODEL_ID, image_dir=N6_IMG_DIR, load_model=N6_LOAD_MODEL, save_model=N6_SAVE_MODEL, write_image=N6_WRITE_IMAGES, batch_size=N6_BATCH_SIZE, total_steps=N6_TOTAL_STEPS, learning_rate=N6_LEARNING_RATE, other_params={}): # Load and separate datasets data = h5py.File(datafile) train_dataset = np.transpose(data.get('train_inputs')) train_labels = np.transpose(data.get('train_labels')) valid_dataset = np.transpose(data.get('valid_inputs')) valid_labels = np.transpose(data.get('valid_labels')) test_dataset = np.transpose(data.get('test_inputs')) test_labels = np.transpose(data.get('test_labels')) kx = int(data.get('kx')[0]) ky = int(data.get('ky')[0]) num_inputs = kx * ky num_outputs = kx * ky num_features = other_params.get('num_features', N6_NUM_FEATURES) conv_size = other_params.get('conv_size', N6_CONV_SIZE) fc_units = other_params.get('fc_units', N6_FC_UNITS) activation = other_params.get('activation', N6_ACTIVATION) kernel_file = other_params.get('kernel_file', N6_KERNEL_FILE) # Load kernels kernel_file = other_params.get('kernel_file', N6_KERNEL_FILE) kernel_data = h5py.File(kernel_file) loaded_kernels = np.transpose(kernel_data.get('kernels')) loaded_kernel_size = np.transpose(kernel_data.get('kx')) # Ensure there isn't any monkey business assert loaded_kernel_size == conv_size def weight_variable(shape): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial) def bias_variable(shape): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial) def conv2d(x, W): return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') def max_pool_2x2(x): return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') # Build graph # Using the standard graph with tf.Graph().as_default(): # Network inputs and outputs # TODO worry about (x, y) position input_placeholder = tf.placeholder(tf.float32, shape=[batch_size, num_inputs]) label_placeholder = tf.placeholder(tf.float32, shape=[batch_size, num_inputs]) # Define first convolution with tf.name_scope('conv'): #W_conv = weight_variable([conv_size, conv_size, 1, num_features]) W_conv = tf.Constant(loaded_kernels) b_conv = bias_variable([num_features]) x_image = tf.reshape(input_placeholder, [-1, kx, ky, 1]) h_conv = tf.nn.relu(conv2d(x_image, W_conv) + b_conv) h_pool = max_pool_2x2(h_conv) #tf.histogram_summary("conv_biases", b_conv) #tf.histogram_summary("conv_weights", W_conv) # Define second convolution """ Lets ignore a second convolution right now with tf.name_scope('conv2'): W_conv2 = weight_variable([5, 5, 32, 64]) b_conv2 = bias_variable([64]) h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) h_pool2 = max_pool_2x2(h_conv2) """ # Define a fully connected layer with tf.name_scope('fc1'): dim = (kx // 2) * (ky //2) * num_features # shrink after pooling W_fc = weight_variable([dim, fc_units]) b_fc = bias_variable([fc_units]) h_pool1_flat = tf.reshape(h_pool, [-1, dim]) h_fc = tf.nn.relu(tf.matmul(h_pool1_flat, W_fc) + b_fc) tf.histogram_summary("fc_biases", b_fc) tf.histogram_summary("fc_weights", W_fc) tf.histogram_summary("fc_layer", h_fc) # TODO Could put Dropout here... # Define output layer with tf.name_scope('out_layer'): W_out = weight_variable([fc_units, num_outputs]) b_out = bias_variable([num_outputs]) if activation == 'relu': out_layer = tf.nn.relu(tf.add(tf.matmul(h_fc, W_out), b_out), name='out_layer') elif activation == 'sigmoid': out_layer = tf.sigmoid(tf.add(tf.matmul(h_fc, W_out), b_out), name='out_layer') elif activation == 'linear': out_layer = tf.add(tf.matmul(h_fc, W_out), b_out, name='out_layer') else: raise Exception('Network activation not know: ' + activation) tf.histogram_summary("out_biases", b_fc) tf.histogram_summary("out_weights", W_fc) tf.histogram_summary("out_layer", out_layer) #cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv)) loss = tf.reduce_sum( tf.square(label_placeholder - out_layer), name='loss') train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss) #correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1)) #accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # Collect all summaries together into one operation tf.scalar_summary(loss.op.name, loss) summary_op = tf.merge_all_summaries() saver = tf.train.Saver() sess = tf.Session() summary_writer = tf.train.SummaryWriter(tensorboard_dir, graph_def=sess.graph_def) sess.run(tf.initialize_all_variables()) for step in range(total_steps): offset = (step * batch_size) % (train_labels.shape[0] - batch_size) batch_data = train_dataset[offset : (offset + batch_size), :] batch_labels = train_labels[offset : (offset + batch_size), :] feed_dict = { input_placeholder : batch_data, label_placeholder : batch_labels } _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) if step % 250 == 0: summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) if step % total_steps - 1 == 0: eval_model(step, batch_size, valid_labels, valid_dataset, sess, out_layer, loss, input_placeholder, label_placeholder) """ if i%100 == 0: train_accuracy = accuracy.eval(feed_dict={ x:batch[0], y_: batch[1], keep_prob: 1.0}) print("step %d, training accuracy %g"%(i, train_accuracy)) train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) print("test accuracy %g"%accuracy.eval(feed_dict={ x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})) """ if save_model: save_path = saver.save(sess, save_dir + model_id) print("Model saved in file: %s" % save_path) ## If on my laptop then just write straight to images if write_image: print("Attempting to write images now...") import visTools batch_data, batch_labels, preds = eval_model(step, batch_size, valid_labels, valid_dataset, sess, out_layer, loss, input_placeholder, label_placeholder) visTools.write_preds(batch_data, batch_labels, preds, image_dir, kx)
def runNet( datafile=N2_DATA_FILE, tensorboard_dir=N2_TENSORBOARD_DIR, save_dir=N2_SAVE_DIR, model_id=N2_MODEL_ID, image_dir=N2_IMG_DIR, load_model=N2_LOAD_MODEL, save_model=N2_SAVE_MODEL, write_image=N2_WRITE_IMAGES, batch_size=N2_BATCH_SIZE, total_steps=N2_TOTAL_STEPS, learning_rate=N2_LEARNING_RATE, other_params={}, ): # Load and separate datasets # data = scipy.io.loadmat(datafile) data = h5py.File(datafile) train_dataset = np.transpose(data.get("train_inputs")) train_labels = np.transpose(data.get("train_labels")) valid_dataset = np.transpose(data.get("valid_inputs")) valid_labels = np.transpose(data.get("valid_labels")) test_dataset = np.transpose(data.get("test_inputs")) test_labels = np.transpose(data.get("test_labels")) # Get input size kx = int(data.get("kx")[0]) ky = int(data.get("ky")[0]) assert kx == ky # NETWORK PARAMS image_size = kx num_inputs = image_size * image_size num_outputs = image_size * image_size # Build graph # Using the standard graph with tf.Graph().as_default(): # Network inputs and outputs input_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_inputs)) label_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_outputs)) # Define hidden layer with tf.name_scope("hidden"): weights = tf.Variable( tf.truncated_normal([num_inputs, num_hidden], stddev=1.0 / num_hidden), name="hid_weights" ) biases = tf.Variable(tf.truncated_normal([num_hidden], stddev=1.0 / num_hidden), name="hid_biases") hidden = tf.nn.relu(tf.matmul(input_placeholder, weights) + biases) # tf.image_summary("hid_weights", weights.reshape([1, TOTAL_PIXELS, HIDDEN_UNITS, 1])) tf.histogram_summary("hid_biases", biases) tf.histogram_summary("hid_weights", weights) tf.histogram_summary("hidden", hidden) # Define output layer with tf.name_scope("out_layer"): weights = tf.Variable( tf.truncated_normal([num_hidden, num_outputs], stddev=1.0 / num_hidden), name="out_weights" ) biases = tf.Variable(tf.truncated_normal([num_outputs], stddev=1.0 / num_hidden), name="out_biases") logits = tf.matmul(hidden, weights) + biases tf.histogram_summary("out_biases", biases) tf.histogram_summary("out_weights", weights) tf.histogram_summary("logits", logits) # loss function - Sum squared difference (well mean...) loss = tf.div(tf.reduce_sum(tf.square(label_placeholder - logits)), num_outputs * batch_size, name="loss") # loss = tf.reduce_sum(tf.square(label_placeholder - logits), name='loss') """ # Linear Loss - sum( (t - a)^2 * lamb(t) ) with tf.name_scope('loss_layer'): with tf.name_scope('lamb'): # Computer L(actual) = actual * m + c g = float(10) # Grayness (or whiteness) of the scene m = tf.constant( (num_outputs - (2.0 * g)) / num_outputs, dtype=tf.float32 ) c = tf.constant( float(g) / num_outputs, dtype=tf.float32) lamb = tf.mul(m, logits) + c squr = tf.square(label_placeholder - logits, name='squr') tmp = tf.mul(squr, lamb, name='tmp') loss = tf.reduce_mean(tf.div(tmp + 1e-9, num_outputs), name='loss') tf.histogram_summary('tmp', tmp) tf.histogram_summary('lamb', lamb) tf.histogram_summary('squr', squr) """ # Partwise loss - sum(relu(l - p) * ~1) + sum(relu(p - l) * ~0) # If it doesn't guess a white highenough its a big loss but if it mislabels # a 0, its not a big deal """ with tf.name_scope('loss_layer'): g = float(10) wc = tf.constant( (TOTAL_PIXELS - g) / float(TOTAL_PIXELS) ) bc = tf.constant( g / TOTAL_PIXELS ) ww = tf.reduce_sum(tf.mul(tf.nn.relu(tf.sub(label_placeholder, logits)), wc)) # Wrong white wb = tf.reduce_sum(tf.mul(tf.nn.relu(tf.sub(logits, label_placeholder)), bc)) # wrong black loss = tf.add(ww, wb, name='loss') tf.histogram_summary('ww', ww) tf.histogram_summary('wb', wb) """ # Log data tf.scalar_summary(loss.op.name, loss) # Training operations optimiser = tf.train.GradientDescentOptimizer(learning_rate) global_step = tf.Variable(0, name="global_step", trainable=False) train_op = optimiser.minimize(loss, global_step=global_step) # Collect all summaries together into one operation summary_op = tf.merge_all_summaries() # Saver will save the state of the network in case of crash etc. saver = tf.train.Saver() # Run graph sess = tf.Session() # Initialise variables init = tf.initialize_all_variables() sess.run(init) if tf.__version__ == "0.7.0": # Bug with tf version on cluster summary_writer = tf.train.SummaryWriter(tensorboard_dir) else: summary_writer = tf.train.SummaryWriter(tensorboard_dir, graph_def=sess.graph_def) for step in range(total_steps): start_time = time.time() # Create feed dictionary offset = (step * batch_size) % (train_labels.shape[0] - batch_size) batch_data = train_dataset[offset : (offset + batch_size), :] batch_labels = train_labels[offset : (offset + batch_size), :] feed_dict = {input_placeholder: batch_data, label_placeholder: batch_labels} """ ## DEBUGGING ################ # Just compute the loss loss_value = sess.run([loss], feed_dict=feed_dict) # THen write the loss and other stats to file summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) continue # Force loop restart ################# END DEBUG ################ """ _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time if step % 250 == 0: summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) # Save a checkpoint and evaluate the model periodically. if step % total_steps - 1 == 0: eval_model( step, batch_size, valid_labels, valid_dataset, sess, logits, loss, input_placeholder, label_placeholder, ) if save_model: save_path = saver.save(sess, save_dir + model_id) print("Model saved in file: %s" % save_path) ## If on my laptop then just write straight to images if write_image: print("Attempting to write images now...") import visTools preds = eval_model( step, batch_size, valid_labels, valid_dataset, sess, logits, loss, input_placeholder, label_placeholder ) visTools.write_preds(batch_data, batch_labels, preds, image_dir, kx)
def runNet(datafile=N5_DATA_FILE, tensorboard_dir=N5_TENSORBOARD_DIR, save_dir=N5_SAVE_DIR, model_id=N5_MODEL_ID, image_dir=N5_IMG_DIR, load_model=N5_LOAD_MODEL, save_model=N5_SAVE_MODEL, write_image=N5_WRITE_IMAGES, batch_size=N5_BATCH_SIZE, total_steps=N5_TOTAL_STEPS, learning_rate=N5_LEARNING_RATE, other_params={}): # Load and separate datasets data = h5py.File(datafile) train_dataset = np.transpose(data.get('train_inputs')) train_labels = np.transpose(data.get('train_labels')) valid_dataset = np.transpose(data.get('valid_inputs')) valid_labels = np.transpose(data.get('valid_labels')) test_dataset = np.transpose(data.get('test_inputs')) test_labels = np.transpose(data.get('test_labels')) #Get input size kx = int(data.get('kx')[0]) ky = int(data.get('ky')[0]) num_hidden_units = other_params.get('num_hidden_units', N5_NUM_HIDDEN_UNITS) activation = other_params.get('activation', N5_ACTIVATION) # NETWORK PARAMS num_input_units = kx * ky #HIDDEN_UNITS = 16 num_output_units = kx * ky print("INPUT_UNITS:", num_input_units, "HIDDEN_UNITS:", num_hidden_units, "OUTPUT_UNITS:", num_output_units) # Build graph # Using the standard graph with tf.Graph().as_default(): # Network inputs and outputs input_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_input_units)) label_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_output_units)) # Define hidden layer with tf.name_scope('hidden_layer'): h_weights = tf.Variable( tf.truncated_normal([num_input_units, num_hidden_units], stddev=0.1), name='h_weights') h_biases = tf.Variable(tf.truncated_normal([num_hidden_units], stddev=0.1), name='h_biases') h_logits = tf.matmul(input_placeholder, h_weights) + h_biases # Hidden layer activations? if activation == 'relu': h_output = tf.nn.relu(h_logits, name='h_output') elif activation == 'sigmoid': h_output = tf.sigmoid(h_logits, name='h_output') elif activation == 'linear': h_output = h_logits else: raise Exception('Activation not known: ' + activation) tf.histogram_summary("h_biases", h_biases) tf.histogram_summary("h_weights", h_weights) tf.histogram_summary("h_logits", h_logits) tf.histogram_summary("h_output", h_output) # Define output layer with tf.name_scope('out_layer'): o_weights = tf.Variable( tf.truncated_normal([num_hidden_units, num_output_units], stddev=0.1), name='o_weights') o_biases = tf.Variable(tf.truncated_normal([num_output_units], stddev=0.1), name='o_biases') o_logits = tf.matmul(h_output, o_weights) + o_biases tf.histogram_summary("out_biases", o_biases) tf.histogram_summary("out_weights", o_weights) tf.histogram_summary("out_logits", o_logits) # loss function - Sum squared difference (well mean...) loss = tf.reduce_mean( tf.square(label_placeholder - o_logits), name='loss') # Log data tf.scalar_summary(loss.op.name, loss) # Training operations optimiser = tf.train.GradientDescentOptimizer(learning_rate) global_step = tf.Variable(0, name='global_step', trainable=False) train_op = optimiser.minimize(loss, global_step=global_step) # Collect all summaries together into one operation summary_op = tf.merge_all_summaries() # Saver will save the state of the network in case of crash etc. saver = tf.train.Saver() # Run graph sess = tf.Session() # Initialise variables if not load_model: init = tf.initialize_all_variables() sess.run(init) else: saver.restore(sess, save_dir + model_id) print("Restored network from file: %s" % save_dir + model_id) summary_writer = tf.train.SummaryWriter(tensorboard_dir, graph_def=sess.graph_def) for step in xrange(total_steps): # Create feed dictionary offset = (step * batch_size) % (train_labels.shape[0] - batch_size) batch_data = train_dataset[offset : (offset + batch_size), :] batch_labels = train_labels[offset : (offset + batch_size), :] feed_dict = { input_placeholder : batch_data, label_placeholder : batch_labels } _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) if step % 250 == 0: summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) # Save a checkpoint and evaluate the model periodically. if step % total_steps - 1 == 0: eval_model(step, batch_size, valid_labels, valid_dataset, sess, o_logits, loss, input_placeholder, label_placeholder) """ if step % total_steps - 1 == 0: # RUN Network with validation data offset = (step * batch_size) % (valid_labels.shape[0] - batch_size) batch_data = valid_dataset[offset : (offset + batch_size), :] batch_labels = valid_labels[offset : (offset + batch_size), :] feed_dict = { input_placeholder : batch_data, label_placeholder : batch_labels } preds, validation_value, = sess.run([logits, loss], feed_dict=feed_dict) print("Step: %d: validation: %.5f" % (step, validation_value)) """ if save_model: save_path = saver.save(sess, save_dir + model_id) print("Model saved in file: %s" % save_path) ## If on my laptop then just write straight to images if write_image: print("Attempting to write images now...") import visTools batch_data, batch_labels, preds = eval_model(step, batch_size, valid_labels, valid_dataset, sess, o_logits, loss, input_placeholder, label_placeholder) visTools.write_preds(batch_data, batch_labels, preds, image_dir, kx)
def runNet(datafile=N2_DATA_FILE, tensorboard_dir=N2_TENSORBOARD_DIR, save_dir=N2_SAVE_DIR, model_id=N2_MODEL_ID, image_dir=N2_IMG_DIR, load_model=N2_LOAD_MODEL, save_model=N2_SAVE_MODEL, write_image=N2_WRITE_IMAGES, batch_size=N2_BATCH_SIZE, total_steps=N2_TOTAL_STEPS, learning_rate=N2_LEARNING_RATE, other_params={}): # Load and separate datasets #data = scipy.io.loadmat(datafile) data = h5py.File(datafile) train_dataset = np.transpose(data.get('train_inputs')) train_labels = np.transpose(data.get('train_labels')) valid_dataset = np.transpose(data.get('valid_inputs')) valid_labels = np.transpose(data.get('valid_labels')) test_dataset = np.transpose(data.get('test_inputs')) test_labels = np.transpose(data.get('test_labels')) #Get input size kx = int(data.get('kx')[0]) ky = int(data.get('ky')[0]) assert kx == ky # NETWORK PARAMS image_size = kx num_inputs = image_size * image_size num_outputs = image_size * image_size # Build graph # Using the standard graph with tf.Graph().as_default(): # Network inputs and outputs input_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_inputs)) label_placeholder = tf.placeholder(tf.float32, shape=(batch_size, num_outputs)) # Define hidden layer with tf.name_scope('hidden'): weights = tf.Variable(tf.truncated_normal([num_inputs, num_hidden], stddev=1.0 / num_hidden), name='hid_weights') biases = tf.Variable(tf.truncated_normal([num_hidden], stddev=1.0 / num_hidden), name='hid_biases') hidden = tf.nn.relu(tf.matmul(input_placeholder, weights) + biases) #tf.image_summary("hid_weights", weights.reshape([1, TOTAL_PIXELS, HIDDEN_UNITS, 1])) tf.histogram_summary("hid_biases", biases) tf.histogram_summary("hid_weights", weights) tf.histogram_summary("hidden", hidden) # Define output layer with tf.name_scope('out_layer'): weights = tf.Variable( tf.truncated_normal([num_hidden, num_outputs], stddev=1.0 / num_hidden), name='out_weights') biases = tf.Variable(tf.truncated_normal([num_outputs], stddev=1.0 / num_hidden), name='out_biases') logits = tf.matmul(hidden, weights) + biases tf.histogram_summary("out_biases", biases) tf.histogram_summary("out_weights", weights) tf.histogram_summary("logits", logits) # loss function - Sum squared difference (well mean...) loss = tf.div(tf.reduce_sum( tf.square(label_placeholder - logits)), num_outputs*batch_size, name='loss') #loss = tf.reduce_sum(tf.square(label_placeholder - logits), name='loss') """ # Linear Loss - sum( (t - a)^2 * lamb(t) ) with tf.name_scope('loss_layer'): with tf.name_scope('lamb'): # Computer L(actual) = actual * m + c g = float(10) # Grayness (or whiteness) of the scene m = tf.constant( (num_outputs - (2.0 * g)) / num_outputs, dtype=tf.float32 ) c = tf.constant( float(g) / num_outputs, dtype=tf.float32) lamb = tf.mul(m, logits) + c squr = tf.square(label_placeholder - logits, name='squr') tmp = tf.mul(squr, lamb, name='tmp') loss = tf.reduce_mean(tf.div(tmp + 1e-9, num_outputs), name='loss') tf.histogram_summary('tmp', tmp) tf.histogram_summary('lamb', lamb) tf.histogram_summary('squr', squr) """ # Partwise loss - sum(relu(l - p) * ~1) + sum(relu(p - l) * ~0) # If it doesn't guess a white highenough its a big loss but if it mislabels # a 0, its not a big deal """ with tf.name_scope('loss_layer'): g = float(10) wc = tf.constant( (TOTAL_PIXELS - g) / float(TOTAL_PIXELS) ) bc = tf.constant( g / TOTAL_PIXELS ) ww = tf.reduce_sum(tf.mul(tf.nn.relu(tf.sub(label_placeholder, logits)), wc)) # Wrong white wb = tf.reduce_sum(tf.mul(tf.nn.relu(tf.sub(logits, label_placeholder)), bc)) # wrong black loss = tf.add(ww, wb, name='loss') tf.histogram_summary('ww', ww) tf.histogram_summary('wb', wb) """ # Log data tf.scalar_summary(loss.op.name, loss) # Training operations optimiser = tf.train.GradientDescentOptimizer(learning_rate) global_step = tf.Variable(0, name='global_step', trainable=False) train_op = optimiser.minimize(loss, global_step=global_step) # Collect all summaries together into one operation summary_op = tf.merge_all_summaries() # Saver will save the state of the network in case of crash etc. saver = tf.train.Saver() # Run graph sess = tf.Session() # Initialise variables init = tf.initialize_all_variables() sess.run(init) if tf.__version__ == '0.7.0': # Bug with tf version on cluster summary_writer = tf.train.SummaryWriter(tensorboard_dir) else: summary_writer = tf.train.SummaryWriter(tensorboard_dir, graph_def=sess.graph_def) for step in range(total_steps): start_time = time.time() # Create feed dictionary offset = (step * batch_size) % (train_labels.shape[0] - batch_size) batch_data = train_dataset[offset : (offset + batch_size), :] batch_labels = train_labels[offset : (offset + batch_size), :] feed_dict = { input_placeholder : batch_data, label_placeholder : batch_labels } """ ## DEBUGGING ################ # Just compute the loss loss_value = sess.run([loss], feed_dict=feed_dict) # THen write the loss and other stats to file summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) continue # Force loop restart ################# END DEBUG ################ """ _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict) duration = time.time() - start_time if step % 250 == 0: summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) # Save a checkpoint and evaluate the model periodically. if step % total_steps - 1 == 0: eval_model(step, batch_size, valid_labels, valid_dataset, sess, logits, loss, input_placeholder, label_placeholder) if save_model: save_path = saver.save(sess, save_dir + model_id) print("Model saved in file: %s" % save_path) ## If on my laptop then just write straight to images if write_image: print("Attempting to write images now...") import visTools preds = eval_model(step, batch_size, valid_labels, valid_dataset, sess, logits, loss, input_placeholder, label_placeholder) visTools.write_preds(batch_data, batch_labels, preds, image_dir, kx)
sess.run(cost, feed_dict={ X: teX[test_indices], Y: teY[test_indices] })) print(i, error[-1]) test_indices = np.arange(len(teX)) # Get A Test Batch np.random.shuffle(test_indices) test_indices = test_indices[0:test_size] pred = sess.run(y, feed_dict={X: teX[test_indices], Y: teY[test_indices]}) ## If on my laptop then just write straight to images if write_image: print("Attempting to write images now...") import visTools image_dir = 'imgs/' kx = DVSX visTools.write_preds(teX[test_indices], teY[test_indices], pred, image_dir, kx) """ # Plot the prediction plt.matshow(pred[1].reshape(img_width, img_height)) # Analysis plt.matshow(pred[0].reshape(img_width, img_height)) plt.figure() plt.plot(error) plt.title('RMSE during training') """