if (i + 1) % 10 == 0:
if params.use_category_normal:
vxx, vyy = data.load_batch_category_normal('val')
else:
vxx, vyy = data.load_batch('val')
t_loss = loss.eval(feed_dict={
model.x: txx,
model.y_: tyy,
model.keep_prob: 1.0
})
v_loss = loss.eval(feed_dict={
model.x: vxx,
model.y_: vyy,
model.keep_prob: 1.0
})
print("step {} of {}, train loss {}, val loss {}".format(
i + 1, params.training_steps, t_loss, v_loss))
if (i + 1) % 100 == 0:
if not os.path.exists(params.save_dir):
os.makedirs(params.save_dir)
checkpoint_path = os.path.join(params.save_dir, "model.ckpt")
filename = saver.save(sess, checkpoint_path)
time_passed = cm.pretty_running_time(time_start)
time_left = cm.pretty_time_left(time_start, i, params.training_steps)
print('Model saved. Time passed: {}. Time left: {}'.format(
time_passed, time_left))
if params.shuffle_training:
data.load_imgs()
for i in xrange(params.training_steps):
txx, tyy = data.load_batch('train')
train_step.run(feed_dict={model.x:txx, model.y_:tyy, model.keep_prob: 0.8})
# write logs at every iteration
if write_summary:
summary = merged_summary_op.eval(feed_dict={model.x: txx, model.y_: tyy, model.keep_prob: 1.0})
#summary_writer.add_summary(summary, i)
if (i+1) % 10 == 0:
vxx, vyy = data.load_batch('val')
t_loss = loss.eval(feed_dict={model.x: txx, model.y_: tyy, model.keep_prob: 1.0})
v_loss = loss.eval(feed_dict={model.x: vxx, model.y_: vyy, model.keep_prob: 1.0})
print "step {} of {}, train loss {}, val loss {}".format(i+1, params.training_steps, t_loss, v_loss)
if (i+1) % 100 == 0:
if not os.path.exists(params.save_dir):
os.makedirs(params.save_dir)
checkpoint_path = os.path.join(params.save_dir, "model.ckpt")
filename = saver.save(sess, checkpoint_path)
time_passed = cm.pretty_running_time(time_start)
time_left = cm.pretty_time_left(time_start, i, params.training_steps)
print 'Model saved. Time passed: {}. Time left: {}'.format(time_passed, time_left)