with sess.as_default():
# Initialize
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Coordinate the loading of image files.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# Restore
if latest_checkpoint is not None:
print('Restoring from: {}'.format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
print(' done!')
else:
print('No checkpoint found in:', checkpoint_paths)
# Evaluation (epoch=-1 to say that this is an evaluation after training)
for batch in range(batches):
print('Batch: {}/{}'.format(batch, batches), run_id)
res = sess.run(evaluations_ops)
plot_evaluation(res, run_id, epoch="-1_{}".format(batch))
print('Batch Cost: {}'.format(res['cost']))
costs.append(res['cost'])
print('Average cost: {}'.format(sum(costs) / len(costs)))
# Finish off the filename queue coordinator.
coord.request_stop()
coord.join(threads)
global_step = res['global_step']
print_term(
'Cost: {} Global step: {}'.format(res['cost'], global_step),
run_id, res['cost'])
print_term(
'Cost_LowRes: {} Global step: {}'.format(
res['cost_fwd'], global_step), run_id, res['cost_fwd'])
print_term(
'Cost_Ref: {} Global step: {}'.format(res['cost_ref'],
global_step), run_id,
res['cost_ref'])
train_col_writer.add_summary(res['summary'], global_step)
train_fwd_writer.add_summary(res['summary_fwd'], global_step)
train_ref_writer.add_summary(res['summary_ref'], global_step)
# Save the variables to disk
save_path = saver.save(sess, checkpoint_paths, global_step)
print_term("Model saved in: %s" % save_path, run_id)
# Evaluation step on validation
res = sess.run(evaluations_ops)
val_col_writer.add_summary(res['summary'], global_step)
val_fwd_writer.add_summary(res['summary_fwd'], global_step)
val_ref_writer.add_summary(res['summary_ref'], global_step)
plot_evaluation(res, run_id, epoch)
# Finish off the filename queue coordinator.
coord.request_stop()
coord.join(threads)
print('Done training...')
ref = Refinement()
evaluations_ops = evaluation_pipeline(col, fwd_col, ref, val_number_of_images)
summary_writer = metrics_system(run_id, sess)
saver, checkpoint_paths, latest_checkpoint = checkpointing_system(run_id)
with sess.as_default():
# Initialize
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
# Coordinate the loading of image files.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# Restore
if latest_checkpoint is not None:
print('Restoring from: {}'.format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
print(' done!')
else:
print('No checkpoint found in:', checkpoint_paths)
# Evaluation (epoch=-1 to say that this is an evaluation after training)
res = sess.run(evaluations_ops)
print('Cost: {}'.format(res['cost']))
plot_evaluation(res, run_id, epoch=-1, is_eval=True)
# Finish off the filename queue coordinator.
coord.request_stop()
coord.join(threads)