def main(_):
data_sets=reader(patchlength=0,\
maxlength=300,\
embedding_size=100,\
num_verbs=10,\
allinclude=False,\
shorten=False,\
shorten_front=False,\
testflag=False,\
passnum=0,\
dpflag=False)
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
# Define loss and optimizer
y_ = tf.placeholder(tf.int64, [None])
# Build the graph for the deep net
y_conv, keep_prob = deepnn(x)
with tf.name_scope('loss'):
cross_entropy = tf.losses.sparse_softmax_cross_entropy(labels=y_,
logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), y_)
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
data = reader.reader()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(20000):
temp, answer = data.list_tags(50)
if i % 100 == 0:
do_evalfake(sess, eval_correct, data_sets, FLAGS.batch_size,
images_placeholder, labels_placeholder)
train_step.run(feed_dict={x: temp, y_: answer, keep_prob: 0.5})
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
data_sets=reader(patchlength=0,\
maxlength=300,\
embedding_size=100,\
num_verbs=10,\
allinclude=False,\
shorten=False,\
shorten_front=False,\
testflag=False,\
passnum=0,\
dpflag=False)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits, keep_prob = mnist.inference(images_placeholder, FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = mnist.loss(logits, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = mnist.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = mnist.evaluation(logits, labels_placeholder)
# Build the summary Tensor based on the TF collection of Summaries.
summary = tf.summary.merge_all()
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
with tf.Session() as session:
sess.run(init)
if True:
model_file = tf.train.latest_checkpoint(FLAGS.log_dir)
saver.restore(sess, model_file)
# Start the training loop.
start_time = time.time()
for step in xrange(FLAGS.max_steps):
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
inputs, answers = data_sets.list_tags(FLAGS.batch_size,
test=False)
# print(len(inputs),len(inputs[0]),inputs[0])
# input()
inputs2 = []
for i in range(len(inputs)):
inputs2.append(inputs[i] / 255)
# print(len(inputs2),len(inputs2[0]),inputs2[0])
# input()
feed_dict = {
images_placeholder: inputs2,
labels_placeholder: answers,
keep_prob: 0.5
}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value, logi = sess.run([train_op, loss, logits],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
# print(logi)
# print(answers)
for i0 in range(FLAGS.batch_size):
lgans = np.argmax(logi[i0])
if (lgans != answers[i0] and False):
for tt in range(784):
if (tt % 28 == 0): print(' ')
if (inputs[i0][tt] != 0):
print('1', end=' ')
else:
print('0', end=' ')
# print('np',np.argmax(i),answers,answers[i0],'np')
print(lgans, answers[i0])
# Update the events file.
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
if (step + 1) % 500 == 0 or (step + 1) == FLAGS.max_steps:
#print('Training Data Eval:')
do_eval(sess, eval_correct, data_sets, FLAGS.batch_size,
images_placeholder, labels_placeholder, keep_prob)
do_evalfake(sess, eval_correct, data_sets,
FLAGS.batch_size, images_placeholder,
labels_placeholder, logits, keep_prob)
# Save a checkpoint and evaluate the model periodically.
#if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_file = os.path.join(FLAGS.log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
print('saved to', checkpoint_file)
'''
def main(_):
# Import data
data_sets=mnistreader.reader()
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
# Define loss and optimizer
y_ = tf.placeholder(tf.int64, [None])
# Build the graph for the deep net
y_conv, keep_prob = deepnn(x)
with tf.name_scope('loss'):
cross_entropy = tf.losses.sparse_softmax_cross_entropy(
labels=y_, logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), y_)
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
if True:
model_file=tf.train.latest_checkpoint(log_dir)
print(model_file)
saver.restore(sess,model_file)
for i in range(400000):
# if False:
inputs,answers=data_sets.list_tags(batch_size,test=False)
# crossloss,_=sess.run([cross_entropy,train_step],feed_dict={x: inputs, y_: answers, keep_prob: 0.5})
if i % 10 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: inputs, y_: answers, keep_prob: 0.5})
print('step %d, training accuracy %g' % (i, train_accuracy))
if i%100==99:
testpointer=data_sets.pointer
data_sets.pointer=int(data_sets.readlength*5/6)
acc=0
cnt=0
while data_sets.pointer!=data_sets.readlength:
cnt+=1
inputs,answers=data_sets.list_tags(batch_size,test=True)
train_accuracy = accuracy.eval(feed_dict={
x: inputs, y_: answers, keep_prob: 1.0})
acc+=train_accuracy
acc=acc/cnt
data_sets.pointer=testpointer
print('step %d, cnt:%d, test accuracy %g' % (i, cnt, acc))
print('saved to',saver.save(sess,log_dir+'model.ckpt',global_step=i))
train_step.run(feed_dict={x: inputs, y_: answers, keep_prob: 0.5})
# print('test accuracy %g' % accuracy.eval(feed_dict={
# x: inputs, y_: answers, keep_prob: 1.0}))
with open('submission6.csv','w') as f:
f.write('ImageId,Label\n')
data_sets=mnistreaderout.reader()
for step in range(560):
# print(step,data_sets.pointer)
inputs,answers=data_sets.list_tags(50,test=True)
# inputs2=[]
# for i in range(len(inputs)):
# inputs2.append(inputs[i]/255)
feed_dict = { x: inputs, y_: answers, keep_prob:1.0}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
anst=sess.run([y_conv], feed_dict=feed_dict)[0]
for i in range(len(anst)):
f.write(str(data_sets.pointer-batch_size+i+1)+','+str(np.argmax(anst[i]))+'\n')
if(data_sets.pointer>100000000 ):
print('anst:',np.argmax(anst[0]),' gen:',data_sets.pointer,' step:',step)
for i2 in range(784):
if(inputs[0][i2]!=0):
print('1',end=' ');
else:
print(' ',end=' ');
if(i2%28==0): print(' ');
input()
'''
def main(_):
# Import data
data_sets=mnistreader.reader(patchlength=0,\
maxlength=300,\
embedding_size=100,\
num_verbs=10,\
allinclude=False,\
shorten=False,\
shorten_front=False,\
testflag=False,\
passnum=0,\
dpflag=False)
# Create the model
x = tf.placeholder(tf.float32, [None, 784])
# Define loss and optimizer
y_ = tf.placeholder(tf.int64, [None])
# Build the graph for the deep net
y_conv, keep_prob = deepnn(x)
with tf.name_scope('loss'):
y_conv_norm = tf.nn.l2_normalize(y_conv, [1])
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=y_, logits=y_conv_norm)
cross_entropy_mean = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy_mean)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1), y_)
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
try:
model_file = tf.train.latest_checkpoint(log_dir)
print(model_file)
saver.restore(sess, model_file)
except Exception as e:
print(e)
maxacc = 0
minloss = 1000
for i in range(400000):
# if False:
inputs, answers = data_sets.list_tags(batch_size, test=False)
# crossloss,_=sess.run([cross_entropy,train_step],feed_dict={x: inputs, y_: answers, keep_prob: 0.5})
if i % 10 == 0:
train_accuracy, lossop, yc, ycn, ce, cor = sess.run(
[
accuracy, cross_entropy_mean, y_conv, y_conv_norm,
cross_entropy, correct_prediction
],
feed_dict={
x: inputs,
y_: answers,
keep_prob: 0.5
})
print('step %d, training accuracy %g loss: %g' %
(i, train_accuracy, lossop))
print('yconv:', yc[0], len(yc))
print('yconvnorm:', ycn[0], len(ycn))
print('yans:', answers[0])
print('cross:', ce[0])
print('correct:', cor[0])
if i % 100 == 0:
testpointer = data_sets.pointer
data_sets.pointer = int(data_sets.readlength * 5 / 6)
acc = 0
cnt = 0
losstot = 0
while data_sets.pointer != data_sets.readlength:
cnt += 1
inputs, answers = data_sets.list_tags(batch_size,
test=True)
train_accuracy, lossop = sess.run(
[accuracy, cross_entropy],
feed_dict={
x: inputs,
y_: answers,
keep_prob: 1
})
acc += train_accuracy
losstot += lossop
acc = acc / cnt
losstot = losstot / cnt
data_sets.pointer = testpointer
print('step %d, cnt:%d, test accuracy %g maxacc %g loss %g' %
(i, cnt, acc, maxacc, losstot))
if (acc > maxacc or (acc == maxacc and losstot < minloss)):
#if(losstot<minloss):
maxacc = acc
minloss = losstot
print(
'saved to',
saver.save(sess, log_dir + 'model.ckpt',
global_step=i))
elif acc - maxacc < -0.03:
model_file = tf.train.latest_checkpoint(log_dir)
print('reload from:', model_file)
saver.restore(sess, model_file)
train_step.run(feed_dict={x: inputs, y_: answers, keep_prob: 0.5})
# print('test accuracy %g' % accuracy.eval(feed_dict={
# x: inputs, y_: answers, keep_prob: 1.0}))
with open('submission6.csv', 'w') as f:
f.write('ImageId,Label\n')
data_sets = mnistreaderout.reader()
for step in range(560):
# print(step,data_sets.pointer)
inputs, answers = data_sets.list_tags(50, test=True)
# inputs2=[]
# for i in range(len(inputs)):
# inputs2.append(inputs[i]/255)
feed_dict = {x: inputs, y_: answers, keep_prob: 1.0}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
anst = sess.run([y_conv], feed_dict=feed_dict)[0]
for i in range(len(anst)):
f.write(
str(data_sets.pointer - batch_size + i + 1) + ',' +
str(np.argmax(anst[i])) + '\n')
if (data_sets.pointer > 100000000):
print('anst:', np.argmax(anst[0]), ' gen:',
data_sets.pointer, ' step:', step)
for i2 in range(784):
if (inputs[0][i2] != 0):
print('1', end=' ')
else:
print(' ', end=' ')
if (i2 % 28 == 0): print(' ')
input()
'''
def run_training():
"""Train MNIST for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
data_sets=reader(patchlength=0,\
maxlength=300,\
embedding_size=100,\
num_verbs=10,\
allinclude=False,\
shorten=False,\
shorten_front=False,\
testflag=False,\
passnum=0,\
dpflag=False)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = mnist.inference(images_placeholder, FLAGS.hidden1,
FLAGS.hidden2)
# Add the variable initializer Op.
init = tf.global_variables_initializer()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
if True:
model_file = tf.train.latest_checkpoint(FLAGS.log_dir)
saver.restore(sess, model_file)
# Start the training loop.
start_time = time.time()
ans = []
with open('submission.csv', 'w') as f:
f.write('ImageId,Label\n')
count = 0
for step in range(42000):
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
inputs, answers = data_sets.list_tags(FLAGS.batch_size,
test=False)
inputs2 = []
for i in range(len(inputs)):
inputs2.append(inputs[i] / 255)
'''
print(len(inputs2),len(inputs2[0]),inputs2[0])
input()
'''
feed_dict = {
images_placeholder: inputs2,
labels_placeholder: answers
}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
anst = sess.run([logits], feed_dict=feed_dict)[0]
for i in anst:
count += 1
f.write(str(count) + ',' + str(np.argmax(i)) + '\n')
ans.append(np.argmax(i))
if (np.argmax(i) != answers[0]):
for tt in range(784):
if (tt % 28 == 0): print(' ')
if (inputs[0][tt] != 0):
print('1', end=' ')
else:
print('0', end=' ')
print('np', np.argmax(i), answers, answers[0], 'np')
input()
print(step, data_sets.pointer, data_sets.readlength, i,
np.argmax(i), answers, answers[0], 'np')
def main(_):
data_sets = reader()
with tf.Graph().as_default():
imagein = tf.placeholder(tf.float32, shape=(batch_size, input_length))
labelin = tf.placeholder(tf.int32, shape=(batch_size))
with tf.name_scope('reshape'):
re_image = tf.reshape(imagein, [-1, 28, 28, 1])
# First convolutional layer - maps one grayscale image to 32 feature maps.
with tf.name_scope('conv1'):
w1 = weight([5, 5, 1, 32])
b1 = bias([32])
h1 = tf.nn.relu(conv2d(re_image, w1) + b1)
# Pooling layer - downsamples by 2X.
with tf.name_scope('pool1'):
hp1 = tf.nn.max_pool(h1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# Second convolutional layer -- maps 32 feature maps to 64.
with tf.name_scope('conv2'):
w2 = weight([5, 5, 32, 64])
b2 = bias([64])
h2 = tf.nn.relu(conv2d(hp1, w2) + b2)
# Second pooling layer.
with tf.name_scope('pool2'):
hp2 = tf.nn.max_pool(h2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
# is down to 7x7x64 feature maps -- maps this to 1024 features.
with tf.name_scope('fc1'):
w3 = weight([7 * 7 * 64, 1024])
b3 = bias([1024])
hp2f = tf.reshape(hp2, [-1, 7 * 7 * 64])
h3 = tf.nn.relu(tf.matmul(hp2f, w3) + b3)
# Dropout - controls the complexity of the model, prevents co-adaptation of
# features.
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
h3d = tf.nn.dropout(h3, keep_prob)
# Map the 1024 features to 10 classes, one for each digit
with tf.name_scope('fc2'):
w4 = weight([1024, 10])
b4 = bias([10])
y_predict_mid = tf.matmul(h3d, w4) + b4
y_predict = tf.nn.l2_normalize(y_predict_mid, [1])
loss = tf.losses.sparse_softmax_cross_entropy(labels=labelin,
logits=y_predict)
correctcount = tf.reduce_sum(
tf.cast(tf.nn.in_top_k(y_predict, labelin, 1), tf.int32))
optimizer = tf.train.GradientDescentOptimizer(0.001)
trainop = optimizer.minimize(loss)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
try:
model_file = tf.train.latest_checkpoint(log_dir)
saver.restore(sess, model_file)
except Exception as e:
print(e)
start_time = time.time()
for step in range(1000000000):
inputs, answers = data_sets.list_tags(batch_size, test=False)
inputs2 = []
for i in range(len(inputs)):
inputs2.append(inputs[i] / 255)
feed_dict = {
imagein: inputs2,
labelin: answers,
keep_prob: 0.5
}
_, loss_value, ypr, yprmid, cor = sess.run(
[trainop, loss, y_predict, y_predict_mid, correctcount],
feed_dict=feed_dict)
duration = time.time() - start_time
if step % 10 == 0:
print('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
print('ypr', ypr[0])
print('yprmid', yprmid[0])
'''
for i0 in range(FLAGS.batch_size):
lgans=np.argmax(logi[i0])
if(lgans!=answers[i0] and False):
for tt in range(784):
if(tt%28==0): print(' ');
if(inputs[i0][tt]!=0):
print('1',end=' ');
else:
print('0',end=' ');
# print('np',np.argmax(i),answers,answers[i0],'np')
print(lgans,answers[i0])
'''
if step % 50 == 0:
test_acc(sess, correctcount, data_sets, batch_size,
imagein, labelin, keep_prob)
checkpoint_file = os.path.join(log_dir, 'model.ckpt')
saver.save(sess, checkpoint_file, global_step=step)
print('saved to', checkpoint_file)