def main(unused_argv): classes = 1721 # Load training and eval data training, t_labels, validation, v_labels = prep.data_from_base( 'training_data') # t_labels = onehot_labels(t_labels, classes) # v_labels = onehot_labels(v_labels, classes) # Create the Estimator hiragana_classifier = learn.Estimator(model_fn=cnn_model_fn, model_dir="/tmp/kanji_cnn_test2") # Set up logging for predictions tensors_to_log = {"probabilities": "softmax_tensor"} logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=100) # Train the model hiragana_classifier.fit(x=training, y=t_labels, batch_size=50, steps=1000, monitors=[logging_hook]) # Configure the accuracy metric for evaluation metrics = { "accuracy": learn.MetricSpec(metric_fn=tf.metrics.accuracy, prediction_key="classes"), } # Evaluate the model and print results eval_results = hiragana_classifier.evaluate(x=validation, y=v_labels, metrics=metrics) print(eval_results)
def main(_): sess = tf.InteractiveSession() def variable_summaries(var): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) tf.summary.histogram('histogram', var) def write_predictions(): pred_list = [] prediction = tf.argmax(y_conv, 1) length = int(len(validation) / test_batch) for i in range(length): a = i * test_batch pred_list = pred_list + prediction.eval( feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0 }, session=sess).tolist() write_file = str(save_location + run_number + '_predictions.txt') with open(write_file, 'w') as f: for i, pred in enumerate(pred_list): string = str(str(pred) + ' ' + str(val_labels[i]) + '\n') f.write(string) error_gen.make_html(net_name, write_file, 'kanji_dictionary.json', 'validation.json') # get accuracy def get_accuracy(step): tot_acc = 0.0 length = int(len(validation) / test_batch) for i in range(length): a = i * test_batch summary, acc = sess.run( [merged, accuracy], feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0 }) validation_writer.add_summary(summary, step + i) tot_acc += acc tot_acc /= length write_predictions() return tot_acc # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 test_batch = 300 steps = 20000 epochs = 40 kernel_size = 3 learn_rate = 0.0001 net_name = 'cnn_kanji_334' cwd = str(os.getcwd()) save_location = str(cwd + '/tensorflow/cnn_kanji/' + net_name) print(save_location) run_number = '/0' # Import data training, t_labels, validation, val_labels = prep.data_from_base( 'train_val_test_data_32') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(val_labels, classes) print('data imported') # Create the model with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, width * height]) y_ = tf.placeholder(tf.float32, [None, classes]) with tf.name_scope('input_reshape'): x_image = tf.reshape(x, [-1, width, height, 1]) # tf.summary.image('input', x_image, classes) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([kernel_size, kernel_size, 1, 32], "w1") variable_summaries(W_conv1) with tf.name_scope('biases'): b_conv1 = bias_variable([32], "b1") with tf.name_scope('activations'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) variable_summaries(h_conv1) # adding the second convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv2 = weight_variable([kernel_size, kernel_size, 32, 32], "w2") variable_summaries(W_conv2) with tf.name_scope('biases'): b_conv2 = bias_variable([32], "b2") with tf.name_scope('activations'): h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) variable_summaries(h_conv2) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv2) # pool1_img = tf.reshape(h_pool1, [-1,width,height,1]) # tf.summary.image('pool1', pool1_img, classes) # adding the third convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv3 = weight_variable([kernel_size, kernel_size, 32, 64], "w3") variable_summaries(W_conv3) with tf.name_scope('biases'): b_conv3 = bias_variable([64], "b3") with tf.name_scope('activations'): h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) variable_summaries(h_conv3) # adding the fourth convolutional layer with tf.name_scope('conv_layer4'): with tf.name_scope('weights'): W_conv4 = weight_variable([kernel_size, kernel_size, 64, 64], "w4") variable_summaries(W_conv4) with tf.name_scope('biases'): b_conv4 = bias_variable([64], "b4") with tf.name_scope('activations'): h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4) variable_summaries(h_conv4) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv4) # pool2_img = tf.reshape(h_pool2, [-1,width,height,1]) # tf.summary.image('pool2', pool2_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer5'): with tf.name_scope('weights'): W_conv5 = weight_variable([kernel_size, kernel_size, 64, 128], "w5") variable_summaries(W_conv5) with tf.name_scope('biases'): b_conv5 = bias_variable([128], "b5") with tf.name_scope('activations'): h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) variable_summaries(h_conv5) # adding the sixth convolutional layer with tf.name_scope('conv_layer6'): with tf.name_scope('weights'): W_conv6 = weight_variable([kernel_size, kernel_size, 128, 128], "w6") variable_summaries(W_conv6) with tf.name_scope('biases'): b_conv6 = bias_variable([128], "b6") with tf.name_scope('activations'): h_conv6 = tf.nn.relu(conv2d(h_conv5, W_conv6) + b_conv6) variable_summaries(h_conv6) # the third pooling layer with tf.name_scope('pooling3'): h_pool3 = max_pool_2x2(h_conv6) # pool3_img = tf.reshape(h_pool3, [-1,width,height,1]) # tf.summary.image('pool3', pool3_img, classes) # adding the seventh convolutional layer with tf.name_scope('conv_layer7'): with tf.name_scope('weights'): W_conv7 = weight_variable([kernel_size, kernel_size, 128, 256], "w7") variable_summaries(W_conv7) with tf.name_scope('biases'): b_conv7 = bias_variable([256], "b7") with tf.name_scope('activations'): h_conv7 = tf.nn.relu(conv2d(h_pool3, W_conv7) + b_conv7) variable_summaries(h_conv7) # adding the eigth convolutional layer with tf.name_scope('conv_layer8'): with tf.name_scope('weights'): W_conv8 = weight_variable([kernel_size, kernel_size, 256, 256], "w8") variable_summaries(W_conv8) with tf.name_scope('biases'): b_conv8 = bias_variable([256], "b8") with tf.name_scope('activations'): h_conv8 = tf.nn.relu(conv2d(h_conv7, W_conv8) + b_conv8) variable_summaries(h_conv8) # adding the ninth convolutional layer with tf.name_scope('conv_layer9'): with tf.name_scope('weights'): W_conv9 = weight_variable([kernel_size, kernel_size, 256, 256], "w9") variable_summaries(W_conv9) with tf.name_scope('biases'): b_conv9 = bias_variable([256], "b9") with tf.name_scope('activations'): h_conv9 = tf.nn.relu(conv2d(h_conv8, W_conv9) + b_conv9) variable_summaries(h_conv9) # the fourth pooling layer with tf.name_scope('pooling4'): h_pool4 = max_pool_2x2(h_conv9) # pool3_img = tf.reshape(h_pool3, [-1,width,height,1]) # tf.summary.image('pool3', pool3_img, classes) h_pool4_flat = tf.reshape(h_pool4, [-1, 2 * 2 * 256]) #adding the final layer with tf.name_scope('fully_connected1'): W_fc1 = weight_variable([2 * 2 * 256, 1024], "W_fc1") b_fc1 = bias_variable([1024], "b_fc1") with tf.name_scope('weights'): variable_summaries(W_fc1) with tf.name_scope('activations'): h_fc1 = tf.nn.relu(tf.matmul(h_pool4_flat, W_fc1) + b_fc1) variable_summaries(h_fc1) # adding the dropout keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer with tf.name_scope('readout_layer'): with tf.name_scope('weights'): W_fc_read = weight_variable([1024, classes], "w_read") variable_summaries(W_fc_read) b_fc_read = bias_variable([classes], "b_read") with tf.name_scope('activations'): y_conv = tf.matmul(h_fc1_drop, W_fc_read) + b_fc_read variable_summaries(y_conv) # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. with tf.name_scope('cross_entropy'): cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) tf.summary.scalar('cross_entropy', cross_entropy) # global_step = tf.Variable(0, name='global_step', trainable=False) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) # Add ops to save and restore all the variables. merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( save_location + run_number + '/logs/kanji_with_summaries/train', sess.graph) validation_writer = tf.summary.FileWriter( save_location + run_number + '/logs/kanji_with_summaries/validation') tf.global_variables_initializer().run() saver = tf.train.Saver() # if os.path.exists(os.path.join(save_location + run_number)): # saver.restore(sess, save_location + run_number + "/model.ckpt") # write_predictions() epoch = -1 i = -1 # Train # for i in range(steps): while epoch < epochs: i += 1 a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }) train_writer.add_summary(summary, i) if i % 200 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batchx, y_: batchy, keep_prob: 1.0 }) print("step %d, training accuracy %g" % (i, train_accuracy)) save_path = saver.save(sess, save_location + run_number + "/model.ckpt", i) if a < batch_size: epoch += 1 acc = get_accuracy(i) print("epoch %d, validation accuracy %g" % (epoch, acc)) # write_predictions() save_path = saver.save(sess, save_location + run_number + "/model.ckpt", i + 1) train_writer.close() validation_writer.close()
def main(_): sess = tf.InteractiveSession() def variable_summaries(var): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) tf.summary.histogram('histogram', var) # get accuracy def get_accuracy(step): tot_acc = 0.0 length = int(len(validation) / test_batch) for i in range(length): a = i * test_batch summary, acc = sess.run( [merged, accuracy], feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0 }) validation_writer.add_summary(summary, step + i) tot_acc += acc tot_acc /= length return tot_acc # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 test_batch = 533 # number in set = 6396, 6396 / 12 = 533 steps = 5000 epochs = 15 kernel_size = 3 learn_rate = 0.0001 save_location = "/tmp/tensorflow/kanji/kanji_cnn_3" run_number = '/2' # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'train_val_test_data_32') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') # Create the model with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, width * height]) y_ = tf.placeholder(tf.float32, [None, classes]) with tf.name_scope('input_reshape'): x_image = tf.reshape(x, [-1, width, height, 1]) tf.summary.image('input', x_image, classes) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([kernel_size, kernel_size, 1, 32], "w1") variable_summaries(W_conv1) with tf.name_scope('biases'): b_conv1 = bias_variable([32], "b1") with tf.name_scope('activations'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) variable_summaries(h_conv1) # adding the second convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv2 = weight_variable([kernel_size, kernel_size, 32, 32], "w2") variable_summaries(W_conv2) with tf.name_scope('biases'): b_conv2 = bias_variable([32], "b2") with tf.name_scope('activations'): h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) variable_summaries(h_conv2) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv2) # pool1_img = tf.reshape(h_pool1, [-1,width,height,1]) # tf.summary.image('pool1', pool1_img, classes) # adding the third convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv3 = weight_variable([kernel_size, kernel_size, 32, 64], "w3") variable_summaries(W_conv3) with tf.name_scope('biases'): b_conv3 = bias_variable([64], "b3") with tf.name_scope('activations'): h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) variable_summaries(h_conv3) # adding the fourth convolutional layer with tf.name_scope('conv_layer4'): with tf.name_scope('weights'): W_conv4 = weight_variable([kernel_size, kernel_size, 64, 64], "w4") variable_summaries(W_conv4) with tf.name_scope('biases'): b_conv4 = bias_variable([64], "b4") with tf.name_scope('activations'): h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4) variable_summaries(h_conv4) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv4) # pool2_img = tf.reshape(h_pool2, [-1,width,height,1]) # tf.summary.image('pool2', pool2_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer5'): with tf.name_scope('weights'): W_conv5 = weight_variable([kernel_size, kernel_size, 64, 128], "w5") variable_summaries(W_conv5) with tf.name_scope('biases'): b_conv5 = bias_variable([128], "b5") with tf.name_scope('activations'): h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) variable_summaries(h_conv5) # adding the sixth convolutional layer with tf.name_scope('conv_layer6'): with tf.name_scope('weights'): W_conv6 = weight_variable([kernel_size, kernel_size, 128, 128], "w6") variable_summaries(W_conv6) with tf.name_scope('biases'): b_conv6 = bias_variable([128], "b6") with tf.name_scope('activations'): h_conv6 = tf.nn.relu(conv2d(h_conv5, W_conv6) + b_conv6) variable_summaries(h_conv6) # the third pooling layer with tf.name_scope('pooling3'): h_pool3 = max_pool_2x2(h_conv6) # pool3_img = tf.reshape(h_pool3, [-1,width,height,1]) # tf.summary.image('pool3', pool3_img, classes) h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 128]) #adding the final layer with tf.name_scope('fully_connected'): W_fc1 = weight_variable([4 * 4 * 128, 6884], "W_fc1") b_fc1 = bias_variable([6884], "b_fc1") with tf.name_scope('weights'): variable_summaries(W_fc1) with tf.name_scope('activations'): h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1) variable_summaries(h_fc1) # adding the dropout keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer with tf.name_scope('readout_layer'): W_fc3 = weight_variable([6884, classes], "w_read") b_fc3 = bias_variable([classes], "b_read") with tf.name_scope('weights'): variable_summaries(W_fc3) with tf.name_scope('activations'): y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3 variable_summaries(y_conv) # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. with tf.name_scope('cross_entropy'): cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) tf.summary.scalar('cross_entropy', cross_entropy) # global_step = tf.Variable(0, name='global_step', trainable=False) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) # Add ops to save and restore all the variables. merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( save_location + run_number + '/logs/kanji_with_summaries/train', sess.graph) validation_writer = tf.summary.FileWriter( save_location + run_number + '/logs/kanji_with_summaries/validation') tf.global_variables_initializer().run() saver = tf.train.Saver() # if os.path.exists(os.path.join(save_location + run_number)): # saver.restore(sess, save_location + run_number + "/model.ckpt") epoch = -1 i = -1 # Train # for i in range(steps): while epoch < epochs: i += 1 a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }) train_writer.add_summary(summary, i) if i % 200 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batchx, y_: batchy, keep_prob: 1.0 }) print("step %d, training accuracy %g" % (i, train_accuracy)) save_path = saver.save(sess, save_location + run_number + "/model.ckpt") if a < batch_size: epoch += 1 acc = get_accuracy(i) print("epoch %d, validation accuracy %g" % (epoch, acc)) # tot_acc = 0.0 # length = len(validation) # failed = [] # f_labels = [] # for i in range(length): # summary, acc = sess.run([merged, accuracy], feed_dict={ # x: validation[i:i+1], # y_: v_labels[i:i+1], # keep_prob: 1.0}) # failed.append(validation[i]) # f_labels.append(v_labels[i]) # tot_acc += acc # # print('curent %g, total %g'%(acc, tot_acc)) # tot_acc /= length # print("validation accuracy %g"%tot_acc) # # with open('f_labels.txt', 'w') as f: # f.write(f_labels) save_path = saver.save(sess, save_location + run_number + "/model.ckpt") train_writer.close() validation_writer.close()
def main(_): # if tf.gfile.Exists(FLAGS.log_dir): # tf.gfile.DeleteRecursively(FLAGS.log_dir) # tf.gfile.MakeDirs(FLAGS.log_dir) sess = tf.InteractiveSession() def variable_summaries(var): with tf.name_scope('summaries'): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) tf.summary.histogram('histogram', var) # get accuracy def get_accuracy(step): test_batch = 500 tot_acc = 0.0 length = int(len(validation) / test_batch) for i in range(length): a = i*test_batch summary, acc = sess.run([merged, accuracy], feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0}) # res = str('%d_%d'% (step, i)) # print(res) validation_writer.add_summary(summary, step + i) tot_acc += acc tot_acc /= length return tot_acc # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 steps = 5000 learn_rate = 0.0001 save_location = "/tmp/cnn_kanji_log" # Import data training, t_labels, validation, v_labels = prep.data_from_base('training_data') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') # Create the model x = tf.placeholder(tf.float32, [None, width * height]) x_image = tf.reshape(x, [-1,width,height,1]) y_ = tf.placeholder(tf.float32, [None, classes]) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([5, 5, 1, 32], "w1") variable_summaries(W_conv1) b_conv1 = bias_variable([32], "b1") with tf.name_scope('activations'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) variable_summaries(h_conv1) # adding the second convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv2 = weight_variable([5, 5, 32, 32], "w2") variable_summaries(W_conv2) b_conv2 = bias_variable([32], "b2") with tf.name_scope('activations'): h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) variable_summaries(h_conv2) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv2) pool1_img = tf.reshape(h_pool1, [-1,width,height,1]) tf.summary.image('pool1', pool1_img, classes) # adding the third convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv3 = weight_variable([5, 5, 32, 64], "w3") variable_summaries(W_conv3) b_conv3 = bias_variable([64], "b3") with tf.name_scope('activations'): h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) variable_summaries(h_conv3) # adding the fourth convolutional layer with tf.name_scope('conv_layer4'): with tf.name_scope('weights'): W_conv4 = weight_variable([5, 5, 64, 64], "w4") variable_summaries(W_conv4) b_conv4 = bias_variable([64], "b4") with tf.name_scope('activations'): h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4) variable_summaries(h_conv4) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv4) pool2_img = tf.reshape(h_pool2, [-1,width,height,1]) tf.summary.image('pool2', pool2_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer5'): with tf.name_scope('weights'): W_conv5 = weight_variable([5, 5, 64, 64], "w5") variable_summaries(W_conv5) b_conv5 = bias_variable([64], "b5") with tf.name_scope('activations'): h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) variable_summaries(h_conv5) # the third pooling layer with tf.name_scope('pooling3'): h_pool3 = max_pool_2x2(h_conv5) pool3_img = tf.reshape(h_pool3, [-1,width,height,1]) tf.summary.image('pool3', pool3_img, classes) #adding the final layer with tf.name_scope('fully_connected1'): with tf.name_scope('weights'): W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1") variable_summaries(W_fc1) b_fc1 = bias_variable([1024], "b_fc1") h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64]) with tf.name_scope('activations'): h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1) variable_summaries(h_fc1) # adding the dropout keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding another fully connected layer with tf.name_scope('fully_connected2'): with tf.name_scope('weights'): W_fc2 = weight_variable([1024, 1024], "W_fc2") b_fc2 = bias_variable([1024], "b_fc2") with tf.name_scope('activations'): h_fc2 = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2) + b_fc2) # adding the readout layer with tf.name_scope('readout_layer'): with tf.name_scope('weights'): W_fc3 = weight_variable([1024, classes], "w_read") b_fc3 = bias_variable([classes], "b_read") with tf.name_scope('activations'): y_conv = tf.matmul(h_fc2, W_fc3) + b_fc3 # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. with tf.name_scope('cross_entropy'): cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) tf.summary.scalar('cross_entropy', cross_entropy) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) # Add ops to save and restore all the variables. merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter('/tmp/tensorflow/cnn_kanji/logs/kanji_with_summaries/train', sess.graph) validation_writer = tf.summary.FileWriter('/tmp/tensorflow/cnn_kanji/logs/kanji_with_summaries/validation') tf.global_variables_initializer().run() saver = tf.train.Saver() if os.path.exists(os.path.join(save_location)): saver.restore(sess, save_location + "/model.ckpt") epoch = -1 test_batch = 500 # Train for i in range(steps): a = i*batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={x: batchx, y_: batchy, keep_prob: 0.5}) train_writer.add_summary(summary, i) if i%100 == 0: train_accuracy = accuracy.eval(feed_dict={ x:batchx, y_: batchy, keep_prob: 1.0}) print("step %d, training accuracy %g"%(i, train_accuracy)) save_path = saver.save(sess, save_location + "/model.ckpt") if a < batch_size: epoch += 1 acc = get_accuracy(i) print("epoch %d, validation accuracy %g"%(epoch, acc)) acc = get_accuracy(steps) print("validation accuracy %g"%acc) save_path = saver.save(sess, save_location + "/model.ckpt") train_writer.close() test_writer.close()
def main(_): # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 steps = 10000 learn_rate = 0.0001 save_location = "/tmp/cnn_kanji_dataset" # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'training_data') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') # Create the model x = tf.placeholder(tf.float32, [None, width * height]) x_image = tf.reshape(x, [-1, width, height, 1]) # Define loss and optimizer y_ = tf.placeholder(tf.float32, [None, classes]) # adding the first convolutional layer W_conv1 = weight_variable([5, 5, 1, 32], "w1") b_conv1 = bias_variable([32], "b1") h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) # adding the second convolutional layer W_conv2 = weight_variable([5, 5, 32, 32], "w2") b_conv2 = bias_variable([32], "b2") h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) # adding the first pooling layer h_pool1 = max_pool_2x2(h_conv2) # adding the third convolutional layer W_conv3 = weight_variable([5, 5, 32, 64], "w3") b_conv3 = bias_variable([64], "b3") h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) # adding the fourth convolutional layer W_conv4 = weight_variable([5, 5, 64, 64], "w4") b_conv4 = bias_variable([64], "b4") h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4) # the second pooling layer h_pool2 = max_pool_2x2(h_conv4) # adding the fifth convolutional layer W_conv5 = weight_variable([5, 5, 64, 64], "w5") b_conv5 = bias_variable([64], "b5") h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) # the second pooling layer h_pool3 = max_pool_2x2(h_conv5) #adding the final layer W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1") b_fc1 = bias_variable([1024], "b_fc1") h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64]) h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1) # adding the dropout keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding another fully connected layer W_fc2 = weight_variable([1024, 1024], "W_fc2") b_fc2 = bias_variable([1024], "b_fc2") h_fc2 = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2) + b_fc2) # adding the readout layer W_fc3 = weight_variable([1024, classes], "w_read") b_fc3 = bias_variable([classes], "b_read") y_conv = tf.matmul(h_fc2, W_fc3) + b_fc3 # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy) # Add ops to save and restore all the variables. saver = tf.train.Saver() sess = tf.InteractiveSession() # sess = tf_debug.LocalCLIDebugWrapperSession(sess) # sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) sess.run(tf.global_variables_initializer()) if os.path.exists(os.path.join(save_location)): saver.restore(sess, save_location + "/model.ckpt") # Test trained model correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) epoch = -1 test_batch = 500 # Train for i in range(steps): a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] train_step.run(feed_dict={x: batchx, y_: batchy, keep_prob: 0.5}) if i % 100 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batchx, y_: batchy, keep_prob: 1.0 }) print("step %d, training accuracy %g" % (i, train_accuracy)) save_path = saver.save(sess, save_location + "/model.ckpt") if a < batch_size: epoch += 1 acc = 0.0 length = int(len(validation) / test_batch) for i in range(length): a = i * test_batch acc += accuracy.eval( feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0 }) acc /= length print("epoch %d, test accuracy %g" % (epoch, acc)) acc = 0.0 length = int(len(validation) / test_batch) for i in range(length): a = i * test_batch acc += accuracy.eval( feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0 }) acc /= length print("test accuracy %g" % acc) # print("test accuracy %g"%accuracy.eval(feed_dict={ # x: validation, y_: v_labels, keep_prob: 1.0})) save_path = saver.save(sess, save_location + "/model.ckpt")
def main(_): sess = tf.InteractiveSession() def variable_summaries(var): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) tf.summary.histogram('histogram', var) # get accuracy def get_accuracy(step): tot_acc = 0.0 length = int(len(validation) / test_batch) for i in range(length): a = i * test_batch summary, acc = sess.run( [merged, accuracy], feed_dict={ x: validation[a:a + test_batch], y_: v_labels[a:a + test_batch], keep_prob: 1.0 }) validation_writer.add_summary(summary, step + i) tot_acc += acc tot_acc /= length return tot_acc # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 test_batch = 533 # number in set = 6396, 6396 / 12 = 533 steps = 5000 epochs = 15 kernel_size = 3 learn_rate = 0.0001 save_location = "/tmp/tensorflow/kanji/kanji_cnn_4" run_number = '/0' # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'train_val_test_data_32') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') # Create the model with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, width * height]) y_ = tf.placeholder(tf.float32, [None, classes]) with tf.name_scope('input_reshape'): x_image = tf.reshape(x, [-1, width, height, 1]) tf.summary.image('input', x_image, classes) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([kernel_size, kernel_size, 1, 32], "w1") variable_summaries(W_conv1) with tf.name_scope('biases'): b_conv1 = bias_variable([32], "b1") with tf.name_scope('activations'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) variable_summaries(h_conv1) # adding the second convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv2 = weight_variable([kernel_size, kernel_size, 32, 32], "w2") variable_summaries(W_conv2) with tf.name_scope('biases'): b_conv2 = bias_variable([32], "b2") with tf.name_scope('activations'): h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2) variable_summaries(h_conv2) # adding the third convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv3 = weight_variable([kernel_size, kernel_size, 32, 32], "w3") variable_summaries(W_conv3) with tf.name_scope('biases'): b_conv3 = bias_variable([32], "b3") with tf.name_scope('activations'): h_conv3 = tf.nn.relu(conv2d(h_conv2, W_conv3) + b_conv3) variable_summaries(h_conv3) # adding the fourth convolutional layer with tf.name_scope('conv_layer4'): with tf.name_scope('weights'): W_conv4 = weight_variable([kernel_size, kernel_size, 32, 32], "w4") variable_summaries(W_conv4) with tf.name_scope('biases'): b_conv4 = bias_variable([32], "b4") with tf.name_scope('activations'): h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4) variable_summaries(h_conv4) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv4) # pool1_img = tf.reshape(h_pool1, [-1,width,height,1]) # tf.summary.image('pool1', pool1_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer5'): with tf.name_scope('weights'): W_conv5 = weight_variable([kernel_size, kernel_size, 32, 64], "w5") variable_summaries(W_conv5) with tf.name_scope('biases'): b_conv5 = bias_variable([64], "b5") with tf.name_scope('activations'): h_conv5 = tf.nn.relu(conv2d(h_pool1, W_conv5) + b_conv5) variable_summaries(h_conv5) # adding the sixth convolutional layer with tf.name_scope('conv_layer6'): with tf.name_scope('weights'): W_conv6 = weight_variable([kernel_size, kernel_size, 64, 64], "w6") variable_summaries(W_conv6) with tf.name_scope('biases'): b_conv6 = bias_variable([64], "b6") with tf.name_scope('activations'): h_conv6 = tf.nn.relu(conv2d(h_conv5, W_conv6) + b_conv6) variable_summaries(h_conv6) # adding the seventh convolutional layer with tf.name_scope('conv_layer7'): with tf.name_scope('weights'): W_conv7 = weight_variable([kernel_size, kernel_size, 64, 64], "w7") variable_summaries(W_conv7) with tf.name_scope('biases'): b_conv7 = bias_variable([64], "b7") with tf.name_scope('activations'): h_conv7 = tf.nn.relu(conv2d(h_conv6, W_conv7) + b_conv7) variable_summaries(h_conv7) # adding the eigth convolutional layer with tf.name_scope('conv_layer8'): with tf.name_scope('weights'): W_conv8 = weight_variable([kernel_size, kernel_size, 64, 64], "w8") variable_summaries(W_conv8) with tf.name_scope('biases'): b_conv8 = bias_variable([64], "b8") with tf.name_scope('activations'): h_conv8 = tf.nn.relu(conv2d(h_conv7, W_conv8) + b_conv8) variable_summaries(h_conv8) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv8) # pool2_img = tf.reshape(h_pool2, [-1,width,height,1]) # tf.summary.image('pool2', pool2_img, classes) h_pool2_flat = tf.reshape(h_pool2, [-1, 8 * 8 * 64]) #adding the final layer with tf.name_scope('fully_connected'): with tf.name_scope('weights'): W_fc1 = weight_variable([8 * 8 * 64, 3442], "W_fc1") variable_summaries(W_fc1) b_fc1 = bias_variable([3442], "b_fc1") with tf.name_scope('activations'): h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) variable_summaries(h_fc1) # adding the dropout keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer with tf.name_scope('readout_layer'): with tf.name_scope('weights'): W_fc3 = weight_variable([3442, classes], "w_read") variable_summaries(W_fc3) b_fc3 = bias_variable([classes], "b_read") with tf.name_scope('activations'): y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3 variable_summaries(y_conv) # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. with tf.name_scope('cross_entropy'): cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) tf.summary.scalar('cross_entropy', cross_entropy) # global_step = tf.Variable(0, name='global_step', trainable=False) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) # Add ops to save and restore all the variables. merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( save_location + run_number + '/logs/kanji_with_summaries/train', sess.graph) validation_writer = tf.summary.FileWriter( save_location + run_number + '/logs/kanji_with_summaries/validation') tf.global_variables_initializer().run() saver = tf.train.Saver() # if os.path.exists(os.path.join(save_location + run_number)): # saver.restore(sess, save_location + run_number + "/model.ckpt") epoch = -1 i = -1 # Train # for i in range(steps): while epoch < epochs: i += 1 a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }) train_writer.add_summary(summary, i) if i % 200 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batchx, y_: batchy, keep_prob: 1.0 }) print("step %d, training accuracy %g" % (i, train_accuracy)) save_path = saver.save(sess, save_location + run_number + "/model.ckpt", i) if a < batch_size: epoch += 1 acc = get_accuracy(i) print("epoch %d, validation accuracy %g" % (epoch, acc)) # Create randomly initialized embedding weights which will be trained. N = classes # Number of items (classes). D = 200 # Dimensionality of the embedding. embedding_var = tf.Variable(tf.random_normal([N, D]), name='image_embedding') # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. embedding = config.embeddings.add() embedding.sprite.image_path = 'home/workspace/kanji_tests/master.jpg' # Specify the width and height of a single thumbnail. embedding.sprite.single_image_dim.extend([20, 20]) embedding.tensor_name = embedding_var.name # Link this tensor to its metadata file (e.g. labels). # embedding.metadata_path = os.path.join('sprites20/', 'labels.tsv') # Use the same LOG_DIR where you stored your checkpoint. summary_writer = tf.summary.FileWriter(save_location + run_number) # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(summary_writer, config) save_path = saver.save(sess, save_location + run_number + "/model.ckpt", i + 1) train_writer.close() validation_writer.close()
def main(_): if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 steps = 1000 save_location = "/tmp/cnn_kanji_simple" # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'training_data') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') sess = tf.InteractiveSession() # Create the model with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, width * height]) y_ = tf.placeholder(tf.float32, [None, classes]) with tf.name_scope('input_reshape'): x_image = tf.reshape(x, [-1, width, height, 1]) tf.summary.image('input', x_image, classes) # setting up the cnn def weight_variable(shape, nme): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial, name=nme) def bias_variable(shape, nme): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial, name=nme) def conv2d(x, W): return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') def max_pool_2x2(x): return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') def variable_summaries(var): with tf.name_scope('summaries'): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) tf.summary.scalar('max', tf.reduce_max(var)) tf.summary.scalar('min', tf.reduce_min(var)) tf.summary.histogram('histogram', var) with tf.name_scope('stddev'): stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([5, 5, 1, 32], "w1") variable_summaries(W_conv1) with tf.name_scope('biases'): b_conv1 = bias_variable([32], "b1") variable_summaries(b_conv1) with tf.name_scope('activation'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) tf.summary.histogram('activations', h_conv1) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv1) pool1_img = tf.reshape(h_pool1, [-1, width, height, 1]) tf.summary.image('pool1', pool1_img, classes) # adding the third convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv3 = weight_variable([5, 5, 32, 64], "w3") variable_summaries(W_conv3) with tf.name_scope('biases'): b_conv3 = bias_variable([64], "b3") variable_summaries(b_conv3) with tf.name_scope('activation'): h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) tf.summary.histogram('activations', h_conv3) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv3) pool2_img = tf.reshape(h_pool2, [-1, width, height, 1]) tf.summary.image('pool2', pool2_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv5 = weight_variable([5, 5, 64, 64], "w5") variable_summaries(W_conv5) with tf.name_scope('biases'): b_conv5 = bias_variable([64], "b5") variable_summaries(b_conv5) with tf.name_scope('activation'): h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) tf.summary.histogram('activations', h_conv5) # the third pooling layer h_pool3 = max_pool_2x2(h_conv5) #adding the final layer with tf.name_scope('final_layer'): with tf.name_scope('weights'): W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1") variable_summaries(W_fc1) with tf.name_scope('biases'): b_fc1 = bias_variable([1024], "b_fc1") variable_summaries(b_fc1) h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64]) with tf.name_scope('Wx_plus_b'): preactivate = tf.matmul(h_pool3_flat, W_fc1) + b_fc1 tf.summary.histogram('pre_activations', preactivate) h_fc1 = tf.nn.relu(preactivate) tf.summary.histogram('activations', h_fc1) # adding the dropout with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) tf.summary.scalar('dropout_keep_probability', keep_prob) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer with tf.name_scope('readout_layer'): with tf.name_scope('weights'): W_fc3 = weight_variable([1024, classes], "w_read") variable_summaries(W_fc3) with tf.name_scope('biases'): b_fc3 = bias_variable([classes], "b_read") variable_summaries(b_fc3) with tf.name_scope('Wx_plus_b'): y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3 tf.summary.histogram('activations', y_conv) with tf.name_scope('cross_entropy'): # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv) with tf.name_scope('total'): cross_entropy = tf.reduce_mean(diff) tf.summary.scalar('cross_entropy', cross_entropy) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( '/tmp/tensorflow/kanji_simple/logs/kanji_with_summaries/train', sess.graph) test_writer = tf.summary.FileWriter( '/tmp/tensorflow/kanji_simple/logs/kanji_with_summaries/test') tf.global_variables_initializer().run() saver = tf.train.Saver() if os.path.exists(os.path.join(save_location)): saver.restore(sess, save_location + "/model.ckpt") epoch = -1 test_batch = 500 # Train for i in range(steps): a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }) train_writer.add_summary(summary, i) if i % 100 == 0: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }, options=run_options, run_metadata=run_metadata) train_writer.add_run_metadata(run_metadata, 'step%03d' % i) train_writer.add_summary(summary, i) print('Adding run metadata for', i) summary, acc = sess.run([merged, accuracy], feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }) test_writer.add_summary(summary, i) print('Accuracy at step %s: %s' % (i, acc)) save_path = saver.save(sess, save_location + "/model.ckpt") save_path = saver.save(sess, save_location + "/model.ckpt") train_writer.close() test_writer.close()
def main(_): if tf.gfile.Exists(FLAGS.log_dir): tf.gfile.DeleteRecursively(FLAGS.log_dir) tf.gfile.MakeDirs(FLAGS.log_dir) # Hyper-parameters width, height = 32, 32 size = (width, height) classes = 1721 batch_size = 50 steps = 10000 save_location = "/tmp/tensorflow/kanji_simple/1" # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'train_val_test_data_32') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') sess = tf.InteractiveSession() # Create the model with tf.name_scope('input'): x = tf.placeholder(tf.float32, [None, width * height]) y_ = tf.placeholder(tf.float32, [None, classes]) with tf.name_scope('input_reshape'): x_image = tf.reshape(x, [-1, width, height, 1]) tf.summary.image('input', x_image, classes) # setting up the cnn def weight_variable(shape, nme): initial = tf.truncated_normal(shape, stddev=0.1) return tf.Variable(initial, name=nme) def bias_variable(shape, nme): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial, name=nme) def conv2d(x, W): return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME') def max_pool_2x2(x): return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') def variable_summaries(var): with tf.name_scope('summaries'): mean = tf.reduce_mean(var) tf.summary.scalar('mean', mean) tf.summary.scalar('max', tf.reduce_max(var)) tf.summary.scalar('min', tf.reduce_min(var)) tf.summary.histogram('histogram', var) with tf.name_scope('stddev'): stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean))) tf.summary.scalar('stddev', stddev) # adding the first convolutional layer with tf.name_scope('conv_layer1'): with tf.name_scope('weights'): W_conv1 = weight_variable([5, 5, 1, 32], "w1") variable_summaries(W_conv1) with tf.name_scope('biases'): b_conv1 = bias_variable([32], "b1") variable_summaries(b_conv1) with tf.name_scope('activation'): h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) tf.summary.histogram('activations', h_conv1) # adding the first pooling layer with tf.name_scope('pooling1'): h_pool1 = max_pool_2x2(h_conv1) pool1_img = tf.reshape(h_pool1, [-1, width, height, 1]) tf.summary.image('pool1', pool1_img, classes) # adding the third convolutional layer with tf.name_scope('conv_layer2'): with tf.name_scope('weights'): W_conv3 = weight_variable([5, 5, 32, 64], "w3") variable_summaries(W_conv3) with tf.name_scope('biases'): b_conv3 = bias_variable([64], "b3") variable_summaries(b_conv3) with tf.name_scope('activation'): h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3) tf.summary.histogram('activations', h_conv3) # the second pooling layer with tf.name_scope('pooling2'): h_pool2 = max_pool_2x2(h_conv3) pool2_img = tf.reshape(h_pool2, [-1, width, height, 1]) tf.summary.image('pool2', pool2_img, classes) # adding the fifth convolutional layer with tf.name_scope('conv_layer3'): with tf.name_scope('weights'): W_conv5 = weight_variable([5, 5, 64, 64], "w5") variable_summaries(W_conv5) with tf.name_scope('biases'): b_conv5 = bias_variable([64], "b5") variable_summaries(b_conv5) with tf.name_scope('activation'): h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5) tf.summary.histogram('activations', h_conv5) # the third pooling layer h_pool3 = max_pool_2x2(h_conv5) #adding the final layer with tf.name_scope('final_layer'): with tf.name_scope('weights'): W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1") variable_summaries(W_fc1) with tf.name_scope('biases'): b_fc1 = bias_variable([1024], "b_fc1") variable_summaries(b_fc1) h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64]) with tf.name_scope('Wx_plus_b'): preactivate = tf.matmul(h_pool3_flat, W_fc1) + b_fc1 tf.summary.histogram('pre_activations', preactivate) h_fc1 = tf.nn.relu(preactivate) tf.summary.histogram('activations', h_fc1) # adding the dropout with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) tf.summary.scalar('dropout_keep_probability', keep_prob) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer with tf.name_scope('readout_layer'): with tf.name_scope('weights'): W_fc3 = weight_variable([1024, classes], "w_read") variable_summaries(W_fc3) with tf.name_scope('biases'): b_fc3 = bias_variable([classes], "b_read") variable_summaries(b_fc3) with tf.name_scope('Wx_plus_b'): y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3 tf.summary.histogram('activations', y_conv) with tf.name_scope('cross_entropy'): # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv) with tf.name_scope('total'): cross_entropy = tf.reduce_mean(diff) tf.summary.scalar('cross_entropy', cross_entropy) with tf.name_scope('train'): train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # Test trained model with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', accuracy) merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( '/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/train', sess.graph) test_writer = tf.summary.FileWriter( '/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/test') tf.global_variables_initializer().run() saver = tf.train.Saver() # if os.path.exists(os.path.join(save_location)): # saver.restore(sess, save_location + "/model.ckpt") epoch = -1 test_batch = 500 # Train for i in range(steps): a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }) train_writer.add_summary(summary, i) if i % 100 == 0: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() summary, _ = sess.run([merged, train_step], feed_dict={ x: batchx, y_: batchy, keep_prob: 0.5 }, options=run_options, run_metadata=run_metadata) train_writer.add_run_metadata(run_metadata, 'step%03d' % i) train_writer.add_summary(summary, i) print('Adding run metadata for', i) summary, acc = sess.run([merged, accuracy], feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }) test_writer.add_summary(summary, i) print('Accuracy at step %s: %s' % (i, acc)) save_path = saver.save(sess, save_location + "/model.ckpt", i) save_path = saver.save(sess, save_location + "/model.ckpt", steps) summary, acc = sess.run([merged, accuracy], feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }) test_writer.add_summary(summary, i) print('Accuracy at step %s: %s' % (i, acc)) prediction = tf.argmax(y_conv, 1) print( "predictions", prediction.eval(feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }, session=sess)) print( "correct predictions", correct_prediction.eval(feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }, session=sess)) # Create randomly initialized embedding weights which will be trained. N = classes # Number of items (classes). D = 200 # Dimensionality of the embedding. embedding_var = tf.Variable(tf.random_normal([N, D]), name='image_embedding') # Format: tensorflow/tensorboard/plugins/projector/projector_config.proto config = projector.ProjectorConfig() # You can add multiple embeddings. Here we add only one. embedding = config.embeddings.add() embedding.sprite.image_path = 'home/workspace/kanji_tests/sprites20/master.jpg' # Specify the width and height of a single thumbnail. embedding.sprite.single_image_dim.extend([20, 20]) embedding.tensor_name = embedding_var.name # Link this tensor to its metadata file (e.g. labels). # embedding.metadata_path = os.path.join('sprites20/', 'labels.tsv') # Use the same LOG_DIR where you stored your checkpoint. summary_writer = tf.summary.FileWriter(save_location) # The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will # read this file during startup. projector.visualize_embeddings(summary_writer, config) train_writer.close() test_writer.close()
def main(_): # Hyper-parameters width, height = 32, 32 classes = 1721 batch_size = 50 steps = 5000 save_location = "/tmp/cnn_kanji_dataset_simple2" # Import data training, t_labels, validation, v_labels = prep.data_from_base( 'training_data') t_labels = onehot_labels(t_labels, classes) v_labels = onehot_labels(v_labels, classes) print('data imported') # Create the model x = tf.placeholder(tf.float32, [None, width * height]) W = tf.Variable(tf.zeros([width * height, classes])) b = tf.Variable(tf.zeros([classes])) y = tf.matmul(x, W) + b # Define loss and optimizer y_ = tf.placeholder(tf.float32, [None, classes]) # adding in the convolutional layer W_conv1 = weight_variable([5, 5, 1, 32], "w1") b_conv1 = bias_variable([32], "b1") x_image = tf.reshape(x, [-1, width, height, 1]) h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) h_pool1 = max_pool_2x2(h_conv1) # adding the second convolutional layer W_conv2 = weight_variable([5, 5, 32, 64], "w2") b_conv2 = bias_variable([64], "b2") h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) h_pool2 = max_pool_2x2(h_conv2) # adding the third convolutional layer W_conv3 = weight_variable([5, 5, 64, 128], "w3") b_conv3 = bias_variable([128], "b3") h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3) h_pool3 = max_pool_2x2(h_conv3) #adding the final layer W_fc1 = weight_variable([4 * 4 * 128, 1024], "w_flat") b_fc1 = bias_variable([1024], "b_flat") h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 128]) h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1) # adding the dropout keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # adding the readout layer W_fc2 = weight_variable([1024, classes], "w_read") b_fc2 = bias_variable([classes], "b_read") y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 # The raw formulation of cross-entropy, # # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), # reduction_indices=[1])) # # can be numerically unstable. # # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw # outputs of 'y', and then average across the batch. cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # Add ops to save and restore all the variables. saver = tf.train.Saver() sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) if os.path.exists(os.path.join(save_location)): saver.restore(sess, save_location + "/model.ckpt") # Test trained model correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) epoch = -1 # Train for i in range(steps): a = i * batch_size % len(training) batchx = training[a:a + batch_size] batchy = t_labels[a:a + batch_size] train_step.run(feed_dict={x: batchx, y_: batchy, keep_prob: 0.5}) if i % 100 == 0: train_accuracy = accuracy.eval(feed_dict={ x: batchx, y_: batchy, keep_prob: 1.0 }) print("step %d, training accuracy %g" % (i, train_accuracy)) save_path = saver.save(sess, save_location + "/model.ckpt") if a < batch_size: epoch += 1 print("epoch %d, test accuracy %g" % (epoch, accuracy.eval(feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 }))) print("test accuracy %g" % accuracy.eval(feed_dict={ x: validation, y_: v_labels, keep_prob: 1.0 })) save_path = saver.save(sess, save_location + "/model.ckpt")