def main(unused_argv):
classes = 1721
# Load training and eval data
training, t_labels, validation, v_labels = prep.data_from_base(
'training_data')
# t_labels = onehot_labels(t_labels, classes)
# v_labels = onehot_labels(v_labels, classes)
# Create the Estimator
hiragana_classifier = learn.Estimator(model_fn=cnn_model_fn,
model_dir="/tmp/kanji_cnn_test2")
# Set up logging for predictions
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log,
every_n_iter=100)
# Train the model
hiragana_classifier.fit(x=training,
y=t_labels,
batch_size=50,
steps=1000,
monitors=[logging_hook])
# Configure the accuracy metric for evaluation
metrics = {
"accuracy":
learn.MetricSpec(metric_fn=tf.metrics.accuracy,
prediction_key="classes"),
}
# Evaluate the model and print results
eval_results = hiragana_classifier.evaluate(x=validation,
y=v_labels,
metrics=metrics)
print(eval_results)
def main(_):
sess = tf.InteractiveSession()
def variable_summaries(var):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.histogram('histogram', var)
def write_predictions():
pred_list = []
prediction = tf.argmax(y_conv, 1)
length = int(len(validation) / test_batch)
for i in range(length):
a = i * test_batch
pred_list = pred_list + prediction.eval(
feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0
},
session=sess).tolist()
write_file = str(save_location + run_number + '_predictions.txt')
with open(write_file, 'w') as f:
for i, pred in enumerate(pred_list):
string = str(str(pred) + ' ' + str(val_labels[i]) + '\n')
f.write(string)
error_gen.make_html(net_name, write_file, 'kanji_dictionary.json',
'validation.json')
# get accuracy
def get_accuracy(step):
tot_acc = 0.0
length = int(len(validation) / test_batch)
for i in range(length):
a = i * test_batch
summary, acc = sess.run(
[merged, accuracy],
feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0
})
validation_writer.add_summary(summary, step + i)
tot_acc += acc
tot_acc /= length
write_predictions()
return tot_acc
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
test_batch = 300
steps = 20000
epochs = 40
kernel_size = 3
learn_rate = 0.0001
net_name = 'cnn_kanji_334'
cwd = str(os.getcwd())
save_location = str(cwd + '/tensorflow/cnn_kanji/' + net_name)
print(save_location)
run_number = '/0'
# Import data
training, t_labels, validation, val_labels = prep.data_from_base(
'train_val_test_data_32')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(val_labels, classes)
print('data imported')
# Create the model
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, width * height])
y_ = tf.placeholder(tf.float32, [None, classes])
with tf.name_scope('input_reshape'):
x_image = tf.reshape(x, [-1, width, height, 1])
# tf.summary.image('input', x_image, classes)
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([kernel_size, kernel_size, 1, 32], "w1")
variable_summaries(W_conv1)
with tf.name_scope('biases'):
b_conv1 = bias_variable([32], "b1")
with tf.name_scope('activations'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
variable_summaries(h_conv1)
# adding the second convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv2 = weight_variable([kernel_size, kernel_size, 32, 32], "w2")
variable_summaries(W_conv2)
with tf.name_scope('biases'):
b_conv2 = bias_variable([32], "b2")
with tf.name_scope('activations'):
h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)
variable_summaries(h_conv2)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv2)
# pool1_img = tf.reshape(h_pool1, [-1,width,height,1])
# tf.summary.image('pool1', pool1_img, classes)
# adding the third convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([kernel_size, kernel_size, 32, 64], "w3")
variable_summaries(W_conv3)
with tf.name_scope('biases'):
b_conv3 = bias_variable([64], "b3")
with tf.name_scope('activations'):
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
variable_summaries(h_conv3)
# adding the fourth convolutional layer
with tf.name_scope('conv_layer4'):
with tf.name_scope('weights'):
W_conv4 = weight_variable([kernel_size, kernel_size, 64, 64], "w4")
variable_summaries(W_conv4)
with tf.name_scope('biases'):
b_conv4 = bias_variable([64], "b4")
with tf.name_scope('activations'):
h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4)
variable_summaries(h_conv4)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv4)
# pool2_img = tf.reshape(h_pool2, [-1,width,height,1])
# tf.summary.image('pool2', pool2_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer5'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([kernel_size, kernel_size, 64, 128],
"w5")
variable_summaries(W_conv5)
with tf.name_scope('biases'):
b_conv5 = bias_variable([128], "b5")
with tf.name_scope('activations'):
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
variable_summaries(h_conv5)
# adding the sixth convolutional layer
with tf.name_scope('conv_layer6'):
with tf.name_scope('weights'):
W_conv6 = weight_variable([kernel_size, kernel_size, 128, 128],
"w6")
variable_summaries(W_conv6)
with tf.name_scope('biases'):
b_conv6 = bias_variable([128], "b6")
with tf.name_scope('activations'):
h_conv6 = tf.nn.relu(conv2d(h_conv5, W_conv6) + b_conv6)
variable_summaries(h_conv6)
# the third pooling layer
with tf.name_scope('pooling3'):
h_pool3 = max_pool_2x2(h_conv6)
# pool3_img = tf.reshape(h_pool3, [-1,width,height,1])
# tf.summary.image('pool3', pool3_img, classes)
# adding the seventh convolutional layer
with tf.name_scope('conv_layer7'):
with tf.name_scope('weights'):
W_conv7 = weight_variable([kernel_size, kernel_size, 128, 256],
"w7")
variable_summaries(W_conv7)
with tf.name_scope('biases'):
b_conv7 = bias_variable([256], "b7")
with tf.name_scope('activations'):
h_conv7 = tf.nn.relu(conv2d(h_pool3, W_conv7) + b_conv7)
variable_summaries(h_conv7)
# adding the eigth convolutional layer
with tf.name_scope('conv_layer8'):
with tf.name_scope('weights'):
W_conv8 = weight_variable([kernel_size, kernel_size, 256, 256],
"w8")
variable_summaries(W_conv8)
with tf.name_scope('biases'):
b_conv8 = bias_variable([256], "b8")
with tf.name_scope('activations'):
h_conv8 = tf.nn.relu(conv2d(h_conv7, W_conv8) + b_conv8)
variable_summaries(h_conv8)
# adding the ninth convolutional layer
with tf.name_scope('conv_layer9'):
with tf.name_scope('weights'):
W_conv9 = weight_variable([kernel_size, kernel_size, 256, 256],
"w9")
variable_summaries(W_conv9)
with tf.name_scope('biases'):
b_conv9 = bias_variable([256], "b9")
with tf.name_scope('activations'):
h_conv9 = tf.nn.relu(conv2d(h_conv8, W_conv9) + b_conv9)
variable_summaries(h_conv9)
# the fourth pooling layer
with tf.name_scope('pooling4'):
h_pool4 = max_pool_2x2(h_conv9)
# pool3_img = tf.reshape(h_pool3, [-1,width,height,1])
# tf.summary.image('pool3', pool3_img, classes)
h_pool4_flat = tf.reshape(h_pool4, [-1, 2 * 2 * 256])
#adding the final layer
with tf.name_scope('fully_connected1'):
W_fc1 = weight_variable([2 * 2 * 256, 1024], "W_fc1")
b_fc1 = bias_variable([1024], "b_fc1")
with tf.name_scope('weights'):
variable_summaries(W_fc1)
with tf.name_scope('activations'):
h_fc1 = tf.nn.relu(tf.matmul(h_pool4_flat, W_fc1) + b_fc1)
variable_summaries(h_fc1)
# adding the dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
with tf.name_scope('readout_layer'):
with tf.name_scope('weights'):
W_fc_read = weight_variable([1024, classes], "w_read")
variable_summaries(W_fc_read)
b_fc_read = bias_variable([classes], "b_read")
with tf.name_scope('activations'):
y_conv = tf.matmul(h_fc1_drop, W_fc_read) + b_fc_read
variable_summaries(y_conv)
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
tf.summary.scalar('cross_entropy', cross_entropy)
# global_step = tf.Variable(0, name='global_step', trainable=False)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Add ops to save and restore all the variables.
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
save_location + run_number + '/logs/kanji_with_summaries/train',
sess.graph)
validation_writer = tf.summary.FileWriter(
save_location + run_number + '/logs/kanji_with_summaries/validation')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# if os.path.exists(os.path.join(save_location + run_number)):
# saver.restore(sess, save_location + run_number + "/model.ckpt")
# write_predictions()
epoch = -1
i = -1
# Train
# for i in range(steps):
while epoch < epochs:
i += 1
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
})
train_writer.add_summary(summary, i)
if i % 200 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batchx,
y_: batchy,
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuracy))
save_path = saver.save(sess,
save_location + run_number + "/model.ckpt",
i)
if a < batch_size:
epoch += 1
acc = get_accuracy(i)
print("epoch %d, validation accuracy %g" % (epoch, acc))
# write_predictions()
save_path = saver.save(sess, save_location + run_number + "/model.ckpt",
i + 1)
train_writer.close()
validation_writer.close()
def main(_):
sess = tf.InteractiveSession()
def variable_summaries(var):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.histogram('histogram', var)
# get accuracy
def get_accuracy(step):
tot_acc = 0.0
length = int(len(validation) / test_batch)
for i in range(length):
a = i * test_batch
summary, acc = sess.run(
[merged, accuracy],
feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0
})
validation_writer.add_summary(summary, step + i)
tot_acc += acc
tot_acc /= length
return tot_acc
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
test_batch = 533 # number in set = 6396, 6396 / 12 = 533
steps = 5000
epochs = 15
kernel_size = 3
learn_rate = 0.0001
save_location = "/tmp/tensorflow/kanji/kanji_cnn_3"
run_number = '/2'
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'train_val_test_data_32')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
# Create the model
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, width * height])
y_ = tf.placeholder(tf.float32, [None, classes])
with tf.name_scope('input_reshape'):
x_image = tf.reshape(x, [-1, width, height, 1])
tf.summary.image('input', x_image, classes)
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([kernel_size, kernel_size, 1, 32], "w1")
variable_summaries(W_conv1)
with tf.name_scope('biases'):
b_conv1 = bias_variable([32], "b1")
with tf.name_scope('activations'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
variable_summaries(h_conv1)
# adding the second convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv2 = weight_variable([kernel_size, kernel_size, 32, 32], "w2")
variable_summaries(W_conv2)
with tf.name_scope('biases'):
b_conv2 = bias_variable([32], "b2")
with tf.name_scope('activations'):
h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)
variable_summaries(h_conv2)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv2)
# pool1_img = tf.reshape(h_pool1, [-1,width,height,1])
# tf.summary.image('pool1', pool1_img, classes)
# adding the third convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([kernel_size, kernel_size, 32, 64], "w3")
variable_summaries(W_conv3)
with tf.name_scope('biases'):
b_conv3 = bias_variable([64], "b3")
with tf.name_scope('activations'):
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
variable_summaries(h_conv3)
# adding the fourth convolutional layer
with tf.name_scope('conv_layer4'):
with tf.name_scope('weights'):
W_conv4 = weight_variable([kernel_size, kernel_size, 64, 64], "w4")
variable_summaries(W_conv4)
with tf.name_scope('biases'):
b_conv4 = bias_variable([64], "b4")
with tf.name_scope('activations'):
h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4)
variable_summaries(h_conv4)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv4)
# pool2_img = tf.reshape(h_pool2, [-1,width,height,1])
# tf.summary.image('pool2', pool2_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer5'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([kernel_size, kernel_size, 64, 128],
"w5")
variable_summaries(W_conv5)
with tf.name_scope('biases'):
b_conv5 = bias_variable([128], "b5")
with tf.name_scope('activations'):
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
variable_summaries(h_conv5)
# adding the sixth convolutional layer
with tf.name_scope('conv_layer6'):
with tf.name_scope('weights'):
W_conv6 = weight_variable([kernel_size, kernel_size, 128, 128],
"w6")
variable_summaries(W_conv6)
with tf.name_scope('biases'):
b_conv6 = bias_variable([128], "b6")
with tf.name_scope('activations'):
h_conv6 = tf.nn.relu(conv2d(h_conv5, W_conv6) + b_conv6)
variable_summaries(h_conv6)
# the third pooling layer
with tf.name_scope('pooling3'):
h_pool3 = max_pool_2x2(h_conv6)
# pool3_img = tf.reshape(h_pool3, [-1,width,height,1])
# tf.summary.image('pool3', pool3_img, classes)
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 128])
#adding the final layer
with tf.name_scope('fully_connected'):
W_fc1 = weight_variable([4 * 4 * 128, 6884], "W_fc1")
b_fc1 = bias_variable([6884], "b_fc1")
with tf.name_scope('weights'):
variable_summaries(W_fc1)
with tf.name_scope('activations'):
h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1)
variable_summaries(h_fc1)
# adding the dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
with tf.name_scope('readout_layer'):
W_fc3 = weight_variable([6884, classes], "w_read")
b_fc3 = bias_variable([classes], "b_read")
with tf.name_scope('weights'):
variable_summaries(W_fc3)
with tf.name_scope('activations'):
y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3
variable_summaries(y_conv)
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
tf.summary.scalar('cross_entropy', cross_entropy)
# global_step = tf.Variable(0, name='global_step', trainable=False)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Add ops to save and restore all the variables.
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
save_location + run_number + '/logs/kanji_with_summaries/train',
sess.graph)
validation_writer = tf.summary.FileWriter(
save_location + run_number + '/logs/kanji_with_summaries/validation')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# if os.path.exists(os.path.join(save_location + run_number)):
# saver.restore(sess, save_location + run_number + "/model.ckpt")
epoch = -1
i = -1
# Train
# for i in range(steps):
while epoch < epochs:
i += 1
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
})
train_writer.add_summary(summary, i)
if i % 200 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batchx,
y_: batchy,
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuracy))
save_path = saver.save(sess,
save_location + run_number + "/model.ckpt")
if a < batch_size:
epoch += 1
acc = get_accuracy(i)
print("epoch %d, validation accuracy %g" % (epoch, acc))
# tot_acc = 0.0
# length = len(validation)
# failed = []
# f_labels = []
# for i in range(length):
# summary, acc = sess.run([merged, accuracy], feed_dict={
# x: validation[i:i+1],
# y_: v_labels[i:i+1],
# keep_prob: 1.0})
# failed.append(validation[i])
# f_labels.append(v_labels[i])
# tot_acc += acc
# # print('curent %g, total %g'%(acc, tot_acc))
# tot_acc /= length
# print("validation accuracy %g"%tot_acc)
#
# with open('f_labels.txt', 'w') as f:
# f.write(f_labels)
save_path = saver.save(sess, save_location + run_number + "/model.ckpt")
train_writer.close()
validation_writer.close()
def main(_):
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
sess = tf.InteractiveSession()
def variable_summaries(var):
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.histogram('histogram', var)
# get accuracy
def get_accuracy(step):
test_batch = 500
tot_acc = 0.0
length = int(len(validation) / test_batch)
for i in range(length):
a = i*test_batch
summary, acc = sess.run([merged, accuracy], feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0})
# res = str('%d_%d'% (step, i))
# print(res)
validation_writer.add_summary(summary, step + i)
tot_acc += acc
tot_acc /= length
return tot_acc
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
steps = 5000
learn_rate = 0.0001
save_location = "/tmp/cnn_kanji_log"
# Import data
training, t_labels, validation, v_labels = prep.data_from_base('training_data')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
# Create the model
x = tf.placeholder(tf.float32, [None, width * height])
x_image = tf.reshape(x, [-1,width,height,1])
y_ = tf.placeholder(tf.float32, [None, classes])
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
variable_summaries(W_conv1)
b_conv1 = bias_variable([32], "b1")
with tf.name_scope('activations'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
variable_summaries(h_conv1)
# adding the second convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv2 = weight_variable([5, 5, 32, 32], "w2")
variable_summaries(W_conv2)
b_conv2 = bias_variable([32], "b2")
with tf.name_scope('activations'):
h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)
variable_summaries(h_conv2)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv2)
pool1_img = tf.reshape(h_pool1, [-1,width,height,1])
tf.summary.image('pool1', pool1_img, classes)
# adding the third convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([5, 5, 32, 64], "w3")
variable_summaries(W_conv3)
b_conv3 = bias_variable([64], "b3")
with tf.name_scope('activations'):
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
variable_summaries(h_conv3)
# adding the fourth convolutional layer
with tf.name_scope('conv_layer4'):
with tf.name_scope('weights'):
W_conv4 = weight_variable([5, 5, 64, 64], "w4")
variable_summaries(W_conv4)
b_conv4 = bias_variable([64], "b4")
with tf.name_scope('activations'):
h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4)
variable_summaries(h_conv4)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv4)
pool2_img = tf.reshape(h_pool2, [-1,width,height,1])
tf.summary.image('pool2', pool2_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer5'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([5, 5, 64, 64], "w5")
variable_summaries(W_conv5)
b_conv5 = bias_variable([64], "b5")
with tf.name_scope('activations'):
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
variable_summaries(h_conv5)
# the third pooling layer
with tf.name_scope('pooling3'):
h_pool3 = max_pool_2x2(h_conv5)
pool3_img = tf.reshape(h_pool3, [-1,width,height,1])
tf.summary.image('pool3', pool3_img, classes)
#adding the final layer
with tf.name_scope('fully_connected1'):
with tf.name_scope('weights'):
W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1")
variable_summaries(W_fc1)
b_fc1 = bias_variable([1024], "b_fc1")
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64])
with tf.name_scope('activations'):
h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1)
variable_summaries(h_fc1)
# adding the dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding another fully connected layer
with tf.name_scope('fully_connected2'):
with tf.name_scope('weights'):
W_fc2 = weight_variable([1024, 1024], "W_fc2")
b_fc2 = bias_variable([1024], "b_fc2")
with tf.name_scope('activations'):
h_fc2 = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
# adding the readout layer
with tf.name_scope('readout_layer'):
with tf.name_scope('weights'):
W_fc3 = weight_variable([1024, classes], "w_read")
b_fc3 = bias_variable([classes], "b_read")
with tf.name_scope('activations'):
y_conv = tf.matmul(h_fc2, W_fc3) + b_fc3
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Add ops to save and restore all the variables.
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter('/tmp/tensorflow/cnn_kanji/logs/kanji_with_summaries/train', sess.graph)
validation_writer = tf.summary.FileWriter('/tmp/tensorflow/cnn_kanji/logs/kanji_with_summaries/validation')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
if os.path.exists(os.path.join(save_location)):
saver.restore(sess, save_location + "/model.ckpt")
epoch = -1
test_batch = 500
# Train
for i in range(steps):
a = i*batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step], feed_dict={x: batchx, y_: batchy, keep_prob: 0.5})
train_writer.add_summary(summary, i)
if i%100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x:batchx, y_: batchy, keep_prob: 1.0})
print("step %d, training accuracy %g"%(i, train_accuracy))
save_path = saver.save(sess, save_location + "/model.ckpt")
if a < batch_size:
epoch += 1
acc = get_accuracy(i)
print("epoch %d, validation accuracy %g"%(epoch, acc))
acc = get_accuracy(steps)
print("validation accuracy %g"%acc)
save_path = saver.save(sess, save_location + "/model.ckpt")
train_writer.close()
test_writer.close()
def main(_):
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
steps = 10000
learn_rate = 0.0001
save_location = "/tmp/cnn_kanji_dataset"
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'training_data')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
# Create the model
x = tf.placeholder(tf.float32, [None, width * height])
x_image = tf.reshape(x, [-1, width, height, 1])
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, classes])
# adding the first convolutional layer
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
b_conv1 = bias_variable([32], "b1")
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
# adding the second convolutional layer
W_conv2 = weight_variable([5, 5, 32, 32], "w2")
b_conv2 = bias_variable([32], "b2")
h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)
# adding the first pooling layer
h_pool1 = max_pool_2x2(h_conv2)
# adding the third convolutional layer
W_conv3 = weight_variable([5, 5, 32, 64], "w3")
b_conv3 = bias_variable([64], "b3")
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
# adding the fourth convolutional layer
W_conv4 = weight_variable([5, 5, 64, 64], "w4")
b_conv4 = bias_variable([64], "b4")
h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4)
# the second pooling layer
h_pool2 = max_pool_2x2(h_conv4)
# adding the fifth convolutional layer
W_conv5 = weight_variable([5, 5, 64, 64], "w5")
b_conv5 = bias_variable([64], "b5")
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
# the second pooling layer
h_pool3 = max_pool_2x2(h_conv5)
#adding the final layer
W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1")
b_fc1 = bias_variable([1024], "b_fc1")
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1)
# adding the dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding another fully connected layer
W_fc2 = weight_variable([1024, 1024], "W_fc2")
b_fc2 = bias_variable([1024], "b_fc2")
h_fc2 = tf.nn.relu(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
# adding the readout layer
W_fc3 = weight_variable([1024, classes], "w_read")
b_fc3 = bias_variable([classes], "b_read")
y_conv = tf.matmul(h_fc2, W_fc3) + b_fc3
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
sess = tf.InteractiveSession()
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
sess.run(tf.global_variables_initializer())
if os.path.exists(os.path.join(save_location)):
saver.restore(sess, save_location + "/model.ckpt")
# Test trained model
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
epoch = -1
test_batch = 500
# Train
for i in range(steps):
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
train_step.run(feed_dict={x: batchx, y_: batchy, keep_prob: 0.5})
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batchx,
y_: batchy,
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuracy))
save_path = saver.save(sess, save_location + "/model.ckpt")
if a < batch_size:
epoch += 1
acc = 0.0
length = int(len(validation) / test_batch)
for i in range(length):
a = i * test_batch
acc += accuracy.eval(
feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0
})
acc /= length
print("epoch %d, test accuracy %g" % (epoch, acc))
acc = 0.0
length = int(len(validation) / test_batch)
for i in range(length):
a = i * test_batch
acc += accuracy.eval(
feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0
})
acc /= length
print("test accuracy %g" % acc)
# print("test accuracy %g"%accuracy.eval(feed_dict={
# x: validation, y_: v_labels, keep_prob: 1.0}))
save_path = saver.save(sess, save_location + "/model.ckpt")
def main(_):
sess = tf.InteractiveSession()
def variable_summaries(var):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
tf.summary.histogram('histogram', var)
# get accuracy
def get_accuracy(step):
tot_acc = 0.0
length = int(len(validation) / test_batch)
for i in range(length):
a = i * test_batch
summary, acc = sess.run(
[merged, accuracy],
feed_dict={
x: validation[a:a + test_batch],
y_: v_labels[a:a + test_batch],
keep_prob: 1.0
})
validation_writer.add_summary(summary, step + i)
tot_acc += acc
tot_acc /= length
return tot_acc
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
test_batch = 533 # number in set = 6396, 6396 / 12 = 533
steps = 5000
epochs = 15
kernel_size = 3
learn_rate = 0.0001
save_location = "/tmp/tensorflow/kanji/kanji_cnn_4"
run_number = '/0'
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'train_val_test_data_32')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
# Create the model
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, width * height])
y_ = tf.placeholder(tf.float32, [None, classes])
with tf.name_scope('input_reshape'):
x_image = tf.reshape(x, [-1, width, height, 1])
tf.summary.image('input', x_image, classes)
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([kernel_size, kernel_size, 1, 32], "w1")
variable_summaries(W_conv1)
with tf.name_scope('biases'):
b_conv1 = bias_variable([32], "b1")
with tf.name_scope('activations'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
variable_summaries(h_conv1)
# adding the second convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv2 = weight_variable([kernel_size, kernel_size, 32, 32], "w2")
variable_summaries(W_conv2)
with tf.name_scope('biases'):
b_conv2 = bias_variable([32], "b2")
with tf.name_scope('activations'):
h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)
variable_summaries(h_conv2)
# adding the third convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([kernel_size, kernel_size, 32, 32], "w3")
variable_summaries(W_conv3)
with tf.name_scope('biases'):
b_conv3 = bias_variable([32], "b3")
with tf.name_scope('activations'):
h_conv3 = tf.nn.relu(conv2d(h_conv2, W_conv3) + b_conv3)
variable_summaries(h_conv3)
# adding the fourth convolutional layer
with tf.name_scope('conv_layer4'):
with tf.name_scope('weights'):
W_conv4 = weight_variable([kernel_size, kernel_size, 32, 32], "w4")
variable_summaries(W_conv4)
with tf.name_scope('biases'):
b_conv4 = bias_variable([32], "b4")
with tf.name_scope('activations'):
h_conv4 = tf.nn.relu(conv2d(h_conv3, W_conv4) + b_conv4)
variable_summaries(h_conv4)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv4)
# pool1_img = tf.reshape(h_pool1, [-1,width,height,1])
# tf.summary.image('pool1', pool1_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer5'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([kernel_size, kernel_size, 32, 64], "w5")
variable_summaries(W_conv5)
with tf.name_scope('biases'):
b_conv5 = bias_variable([64], "b5")
with tf.name_scope('activations'):
h_conv5 = tf.nn.relu(conv2d(h_pool1, W_conv5) + b_conv5)
variable_summaries(h_conv5)
# adding the sixth convolutional layer
with tf.name_scope('conv_layer6'):
with tf.name_scope('weights'):
W_conv6 = weight_variable([kernel_size, kernel_size, 64, 64], "w6")
variable_summaries(W_conv6)
with tf.name_scope('biases'):
b_conv6 = bias_variable([64], "b6")
with tf.name_scope('activations'):
h_conv6 = tf.nn.relu(conv2d(h_conv5, W_conv6) + b_conv6)
variable_summaries(h_conv6)
# adding the seventh convolutional layer
with tf.name_scope('conv_layer7'):
with tf.name_scope('weights'):
W_conv7 = weight_variable([kernel_size, kernel_size, 64, 64], "w7")
variable_summaries(W_conv7)
with tf.name_scope('biases'):
b_conv7 = bias_variable([64], "b7")
with tf.name_scope('activations'):
h_conv7 = tf.nn.relu(conv2d(h_conv6, W_conv7) + b_conv7)
variable_summaries(h_conv7)
# adding the eigth convolutional layer
with tf.name_scope('conv_layer8'):
with tf.name_scope('weights'):
W_conv8 = weight_variable([kernel_size, kernel_size, 64, 64], "w8")
variable_summaries(W_conv8)
with tf.name_scope('biases'):
b_conv8 = bias_variable([64], "b8")
with tf.name_scope('activations'):
h_conv8 = tf.nn.relu(conv2d(h_conv7, W_conv8) + b_conv8)
variable_summaries(h_conv8)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv8)
# pool2_img = tf.reshape(h_pool2, [-1,width,height,1])
# tf.summary.image('pool2', pool2_img, classes)
h_pool2_flat = tf.reshape(h_pool2, [-1, 8 * 8 * 64])
#adding the final layer
with tf.name_scope('fully_connected'):
with tf.name_scope('weights'):
W_fc1 = weight_variable([8 * 8 * 64, 3442], "W_fc1")
variable_summaries(W_fc1)
b_fc1 = bias_variable([3442], "b_fc1")
with tf.name_scope('activations'):
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
variable_summaries(h_fc1)
# adding the dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
with tf.name_scope('readout_layer'):
with tf.name_scope('weights'):
W_fc3 = weight_variable([3442, classes], "w_read")
variable_summaries(W_fc3)
b_fc3 = bias_variable([classes], "b_read")
with tf.name_scope('activations'):
y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3
variable_summaries(y_conv)
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
with tf.name_scope('cross_entropy'):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
tf.summary.scalar('cross_entropy', cross_entropy)
# global_step = tf.Variable(0, name='global_step', trainable=False)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(learn_rate).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# Add ops to save and restore all the variables.
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
save_location + run_number + '/logs/kanji_with_summaries/train',
sess.graph)
validation_writer = tf.summary.FileWriter(
save_location + run_number + '/logs/kanji_with_summaries/validation')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# if os.path.exists(os.path.join(save_location + run_number)):
# saver.restore(sess, save_location + run_number + "/model.ckpt")
epoch = -1
i = -1
# Train
# for i in range(steps):
while epoch < epochs:
i += 1
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
})
train_writer.add_summary(summary, i)
if i % 200 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batchx,
y_: batchy,
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuracy))
save_path = saver.save(sess,
save_location + run_number + "/model.ckpt",
i)
if a < batch_size:
epoch += 1
acc = get_accuracy(i)
print("epoch %d, validation accuracy %g" % (epoch, acc))
# Create randomly initialized embedding weights which will be trained.
N = classes # Number of items (classes).
D = 200 # Dimensionality of the embedding.
embedding_var = tf.Variable(tf.random_normal([N, D]),
name='image_embedding')
# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto
config = projector.ProjectorConfig()
# You can add multiple embeddings. Here we add only one.
embedding = config.embeddings.add()
embedding.sprite.image_path = 'home/workspace/kanji_tests/master.jpg'
# Specify the width and height of a single thumbnail.
embedding.sprite.single_image_dim.extend([20, 20])
embedding.tensor_name = embedding_var.name
# Link this tensor to its metadata file (e.g. labels).
# embedding.metadata_path = os.path.join('sprites20/', 'labels.tsv')
# Use the same LOG_DIR where you stored your checkpoint.
summary_writer = tf.summary.FileWriter(save_location + run_number)
# The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will
# read this file during startup.
projector.visualize_embeddings(summary_writer, config)
save_path = saver.save(sess, save_location + run_number + "/model.ckpt",
i + 1)
train_writer.close()
validation_writer.close()
def main(_):
if tf.gfile.Exists(FLAGS.log_dir):
tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
steps = 1000
save_location = "/tmp/cnn_kanji_simple"
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'training_data')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
sess = tf.InteractiveSession()
# Create the model
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, width * height])
y_ = tf.placeholder(tf.float32, [None, classes])
with tf.name_scope('input_reshape'):
x_image = tf.reshape(x, [-1, width, height, 1])
tf.summary.image('input', x_image, classes)
# setting up the cnn
def weight_variable(shape, nme):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=nme)
def bias_variable(shape, nme):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial, name=nme)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
def variable_summaries(var):
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
variable_summaries(W_conv1)
with tf.name_scope('biases'):
b_conv1 = bias_variable([32], "b1")
variable_summaries(b_conv1)
with tf.name_scope('activation'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
tf.summary.histogram('activations', h_conv1)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv1)
pool1_img = tf.reshape(h_pool1, [-1, width, height, 1])
tf.summary.image('pool1', pool1_img, classes)
# adding the third convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([5, 5, 32, 64], "w3")
variable_summaries(W_conv3)
with tf.name_scope('biases'):
b_conv3 = bias_variable([64], "b3")
variable_summaries(b_conv3)
with tf.name_scope('activation'):
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
tf.summary.histogram('activations', h_conv3)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv3)
pool2_img = tf.reshape(h_pool2, [-1, width, height, 1])
tf.summary.image('pool2', pool2_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([5, 5, 64, 64], "w5")
variable_summaries(W_conv5)
with tf.name_scope('biases'):
b_conv5 = bias_variable([64], "b5")
variable_summaries(b_conv5)
with tf.name_scope('activation'):
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
tf.summary.histogram('activations', h_conv5)
# the third pooling layer
h_pool3 = max_pool_2x2(h_conv5)
#adding the final layer
with tf.name_scope('final_layer'):
with tf.name_scope('weights'):
W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1")
variable_summaries(W_fc1)
with tf.name_scope('biases'):
b_fc1 = bias_variable([1024], "b_fc1")
variable_summaries(b_fc1)
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64])
with tf.name_scope('Wx_plus_b'):
preactivate = tf.matmul(h_pool3_flat, W_fc1) + b_fc1
tf.summary.histogram('pre_activations', preactivate)
h_fc1 = tf.nn.relu(preactivate)
tf.summary.histogram('activations', h_fc1)
# adding the dropout
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
with tf.name_scope('readout_layer'):
with tf.name_scope('weights'):
W_fc3 = weight_variable([1024, classes], "w_read")
variable_summaries(W_fc3)
with tf.name_scope('biases'):
b_fc3 = bias_variable([classes], "b_read")
variable_summaries(b_fc3)
with tf.name_scope('Wx_plus_b'):
y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3
tf.summary.histogram('activations', y_conv)
with tf.name_scope('cross_entropy'):
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_conv)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1),
tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
'/tmp/tensorflow/kanji_simple/logs/kanji_with_summaries/train',
sess.graph)
test_writer = tf.summary.FileWriter(
'/tmp/tensorflow/kanji_simple/logs/kanji_with_summaries/test')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
if os.path.exists(os.path.join(save_location)):
saver.restore(sess, save_location + "/model.ckpt")
epoch = -1
test_batch = 500
# Train
for i in range(steps):
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
})
train_writer.add_summary(summary, i)
if i % 100 == 0:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
},
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
print('Adding run metadata for', i)
summary, acc = sess.run([merged, accuracy],
feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
})
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
save_path = saver.save(sess, save_location + "/model.ckpt")
save_path = saver.save(sess, save_location + "/model.ckpt")
train_writer.close()
test_writer.close()
def main(_):
if tf.gfile.Exists(FLAGS.log_dir):
tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# Hyper-parameters
width, height = 32, 32
size = (width, height)
classes = 1721
batch_size = 50
steps = 10000
save_location = "/tmp/tensorflow/kanji_simple/1"
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'train_val_test_data_32')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
sess = tf.InteractiveSession()
# Create the model
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, width * height])
y_ = tf.placeholder(tf.float32, [None, classes])
with tf.name_scope('input_reshape'):
x_image = tf.reshape(x, [-1, width, height, 1])
tf.summary.image('input', x_image, classes)
# setting up the cnn
def weight_variable(shape, nme):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=nme)
def bias_variable(shape, nme):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial, name=nme)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
def variable_summaries(var):
with tf.name_scope('summaries'):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean)
tf.summary.scalar('max', tf.reduce_max(var))
tf.summary.scalar('min', tf.reduce_min(var))
tf.summary.histogram('histogram', var)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev)
# adding the first convolutional layer
with tf.name_scope('conv_layer1'):
with tf.name_scope('weights'):
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
variable_summaries(W_conv1)
with tf.name_scope('biases'):
b_conv1 = bias_variable([32], "b1")
variable_summaries(b_conv1)
with tf.name_scope('activation'):
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
tf.summary.histogram('activations', h_conv1)
# adding the first pooling layer
with tf.name_scope('pooling1'):
h_pool1 = max_pool_2x2(h_conv1)
pool1_img = tf.reshape(h_pool1, [-1, width, height, 1])
tf.summary.image('pool1', pool1_img, classes)
# adding the third convolutional layer
with tf.name_scope('conv_layer2'):
with tf.name_scope('weights'):
W_conv3 = weight_variable([5, 5, 32, 64], "w3")
variable_summaries(W_conv3)
with tf.name_scope('biases'):
b_conv3 = bias_variable([64], "b3")
variable_summaries(b_conv3)
with tf.name_scope('activation'):
h_conv3 = tf.nn.relu(conv2d(h_pool1, W_conv3) + b_conv3)
tf.summary.histogram('activations', h_conv3)
# the second pooling layer
with tf.name_scope('pooling2'):
h_pool2 = max_pool_2x2(h_conv3)
pool2_img = tf.reshape(h_pool2, [-1, width, height, 1])
tf.summary.image('pool2', pool2_img, classes)
# adding the fifth convolutional layer
with tf.name_scope('conv_layer3'):
with tf.name_scope('weights'):
W_conv5 = weight_variable([5, 5, 64, 64], "w5")
variable_summaries(W_conv5)
with tf.name_scope('biases'):
b_conv5 = bias_variable([64], "b5")
variable_summaries(b_conv5)
with tf.name_scope('activation'):
h_conv5 = tf.nn.relu(conv2d(h_pool2, W_conv5) + b_conv5)
tf.summary.histogram('activations', h_conv5)
# the third pooling layer
h_pool3 = max_pool_2x2(h_conv5)
#adding the final layer
with tf.name_scope('final_layer'):
with tf.name_scope('weights'):
W_fc1 = weight_variable([4 * 4 * 64, 1024], "W_fc1")
variable_summaries(W_fc1)
with tf.name_scope('biases'):
b_fc1 = bias_variable([1024], "b_fc1")
variable_summaries(b_fc1)
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 64])
with tf.name_scope('Wx_plus_b'):
preactivate = tf.matmul(h_pool3_flat, W_fc1) + b_fc1
tf.summary.histogram('pre_activations', preactivate)
h_fc1 = tf.nn.relu(preactivate)
tf.summary.histogram('activations', h_fc1)
# adding the dropout
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
tf.summary.scalar('dropout_keep_probability', keep_prob)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
with tf.name_scope('readout_layer'):
with tf.name_scope('weights'):
W_fc3 = weight_variable([1024, classes], "w_read")
variable_summaries(W_fc3)
with tf.name_scope('biases'):
b_fc3 = bias_variable([classes], "b_read")
variable_summaries(b_fc3)
with tf.name_scope('Wx_plus_b'):
y_conv = tf.matmul(h_fc1_drop, W_fc3) + b_fc3
tf.summary.histogram('activations', y_conv)
with tf.name_scope('cross_entropy'):
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
diff = tf.nn.softmax_cross_entropy_with_logits(labels=y_,
logits=y_conv)
with tf.name_scope('total'):
cross_entropy = tf.reduce_mean(diff)
tf.summary.scalar('cross_entropy', cross_entropy)
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
# Test trained model
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1),
tf.argmax(y_, 1))
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(
'/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/train',
sess.graph)
test_writer = tf.summary.FileWriter(
'/tmp/tensorflow/kanji_simple/1/logs/kanji_with_summaries/test')
tf.global_variables_initializer().run()
saver = tf.train.Saver()
# if os.path.exists(os.path.join(save_location)):
# saver.restore(sess, save_location + "/model.ckpt")
epoch = -1
test_batch = 500
# Train
for i in range(steps):
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
})
train_writer.add_summary(summary, i)
if i % 100 == 0:
run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
summary, _ = sess.run([merged, train_step],
feed_dict={
x: batchx,
y_: batchy,
keep_prob: 0.5
},
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
train_writer.add_summary(summary, i)
print('Adding run metadata for', i)
summary, acc = sess.run([merged, accuracy],
feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
})
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
save_path = saver.save(sess, save_location + "/model.ckpt", i)
save_path = saver.save(sess, save_location + "/model.ckpt", steps)
summary, acc = sess.run([merged, accuracy],
feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
})
test_writer.add_summary(summary, i)
print('Accuracy at step %s: %s' % (i, acc))
prediction = tf.argmax(y_conv, 1)
print(
"predictions",
prediction.eval(feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
},
session=sess))
print(
"correct predictions",
correct_prediction.eval(feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
},
session=sess))
# Create randomly initialized embedding weights which will be trained.
N = classes # Number of items (classes).
D = 200 # Dimensionality of the embedding.
embedding_var = tf.Variable(tf.random_normal([N, D]),
name='image_embedding')
# Format: tensorflow/tensorboard/plugins/projector/projector_config.proto
config = projector.ProjectorConfig()
# You can add multiple embeddings. Here we add only one.
embedding = config.embeddings.add()
embedding.sprite.image_path = 'home/workspace/kanji_tests/sprites20/master.jpg'
# Specify the width and height of a single thumbnail.
embedding.sprite.single_image_dim.extend([20, 20])
embedding.tensor_name = embedding_var.name
# Link this tensor to its metadata file (e.g. labels).
# embedding.metadata_path = os.path.join('sprites20/', 'labels.tsv')
# Use the same LOG_DIR where you stored your checkpoint.
summary_writer = tf.summary.FileWriter(save_location)
# The next line writes a projector_config.pbtxt in the LOG_DIR. TensorBoard will
# read this file during startup.
projector.visualize_embeddings(summary_writer, config)
train_writer.close()
test_writer.close()
def main(_):
# Hyper-parameters
width, height = 32, 32
classes = 1721
batch_size = 50
steps = 5000
save_location = "/tmp/cnn_kanji_dataset_simple2"
# Import data
training, t_labels, validation, v_labels = prep.data_from_base(
'training_data')
t_labels = onehot_labels(t_labels, classes)
v_labels = onehot_labels(v_labels, classes)
print('data imported')
# Create the model
x = tf.placeholder(tf.float32, [None, width * height])
W = tf.Variable(tf.zeros([width * height, classes]))
b = tf.Variable(tf.zeros([classes]))
y = tf.matmul(x, W) + b
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, classes])
# adding in the convolutional layer
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
b_conv1 = bias_variable([32], "b1")
x_image = tf.reshape(x, [-1, width, height, 1])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
# adding the second convolutional layer
W_conv2 = weight_variable([5, 5, 32, 64], "w2")
b_conv2 = bias_variable([64], "b2")
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
# adding the third convolutional layer
W_conv3 = weight_variable([5, 5, 64, 128], "w3")
b_conv3 = bias_variable([128], "b3")
h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3)
h_pool3 = max_pool_2x2(h_conv3)
#adding the final layer
W_fc1 = weight_variable([4 * 4 * 128, 1024], "w_flat")
b_fc1 = bias_variable([1024], "b_flat")
h_pool3_flat = tf.reshape(h_pool3, [-1, 4 * 4 * 128])
h_fc1 = tf.nn.relu(tf.matmul(h_pool3_flat, W_fc1) + b_fc1)
# adding the dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# adding the readout layer
W_fc2 = weight_variable([1024, classes], "w_read")
b_fc2 = bias_variable([classes], "b_read")
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
# The raw formulation of cross-entropy,
#
# tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
# reduction_indices=[1]))
#
# can be numerically unstable.
#
# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
# outputs of 'y', and then average across the batch.
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
if os.path.exists(os.path.join(save_location)):
saver.restore(sess, save_location + "/model.ckpt")
# Test trained model
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
epoch = -1
# Train
for i in range(steps):
a = i * batch_size % len(training)
batchx = training[a:a + batch_size]
batchy = t_labels[a:a + batch_size]
train_step.run(feed_dict={x: batchx, y_: batchy, keep_prob: 0.5})
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batchx,
y_: batchy,
keep_prob: 1.0
})
print("step %d, training accuracy %g" % (i, train_accuracy))
save_path = saver.save(sess, save_location + "/model.ckpt")
if a < batch_size:
epoch += 1
print("epoch %d, test accuracy %g" %
(epoch,
accuracy.eval(feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
})))
print("test accuracy %g" % accuracy.eval(feed_dict={
x: validation,
y_: v_labels,
keep_prob: 1.0
}))
save_path = saver.save(sess, save_location + "/model.ckpt")
