ensure_dir(current_logs_path)
summary_writer = tf.train.SummaryWriter(os.path.expanduser(current_logs_path), session.graph.as_graph_def())
merged_summary = tf.merge_all_summaries()
session.run(tf.initialize_all_variables())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
window_loss = 20.
processed = 0
try:
while not coord.should_stop():
_, loss, summary = session.run([net.train_task, net.loss, merged_summary])
processed += 1
window_loss = 0.95 * window_loss + 0.05 * loss
logger().info('Batch %d completed, smoothed loss is %f', processed, window_loss)
summary_writer.add_summary(summary, processed)
if processed % CHECKPOINT_INTERVAL == 0:
epochs_completed = processed / CHECKPOINT_INTERVAL
weights_file = config.weights_file_template % epochs_completed
net.save(weights_file)
logger().info('Model checkpoint saved to %s', weights_file)
except tf.errors.OutOfRangeError:
print 'Done'
finally:
coord.request_stop()
coord.join(threads)
session.close()
