def _do_python_evaluation(hypes, step, sess_coll, objective,
image_pl, softmax):
logging.info('Doing Python Evaluation.')
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
eval_dict, images = objective.evaluate(hypes, sess, image_pl, softmax)
utils.print_eval_dict(eval_dict)
_write_eval_dict_to_summary(eval_dict, summary_writer, step)
_write_images_to_summary(images, summary_writer, step)
return
def _do_python_evaluation(hypes, step, sess_coll, objective,
image_pl, softmax):
logging.info('Doing Python Evaluation.')
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
eval_dict, images = objective.evaluate(hypes, sess, image_pl, softmax)
utils.print_eval_dict(eval_dict)
_write_eval_dict_to_summary(eval_dict, summary_writer, step)
_write_images_to_summary(images, summary_writer, step)
return
def do_analyze(logdir, base_path=None):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
if base_path is not None:
hypes['dirs']['base_path'] = base_path
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# prepaire the tv session
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules,
image=image)
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
core.load_weights(logdir, sess, saver)
logging.info("Graph loaded succesfully. Starting evaluation.")
output_dir = os.path.join(logdir, 'analyse')
logging.info("Output Images will be written to: {}".format(
os.path.join(output_dir, "images/")))
logging_file = os.path.join(logdir, "analyse/output.log")
utils.create_filewrite_handler(logging_file)
eval_dict, images = modules['eval'].evaluate(
hypes, sess, image_pl, inf_out)
logging.info("Evaluation Succesfull. Results:")
utils.print_eval_dict(eval_dict)
_write_images_to_logdir(images, output_dir)
def run_evaling(hypes, modules, tv_graph, tv_sess, start_step=0):
"""Run one iteration of training."""
# Unpack operations for later use
summary = tf.Summary()
sess = tv_sess['sess']
summary_writer = tv_sess['writer']
solver = modules['solver']
display_iter = hypes['logging']['display_iter']
write_iter = hypes['logging'].get('write_iter', 5 * display_iter)
eval_iter = hypes['logging']['eval_iter']
save_iter = hypes['logging']['save_iter']
image_iter = hypes['logging'].get('image_iter', 5 * save_iter)
py_smoother = MedianSmoother(20)
step = 0
# Run the training Step
logging.info('Running Evaluation Script.')
eval_dict, images = modules['eval'].evaluate(hypes, sess,
tv_graph['image_pl'],
tv_graph['calib_pl'],
tv_graph['xy_scale_pl'],
tv_graph['inf_out'])
#_write_images_to_summary(images, summary_writer, step)
logging.info("Evaluation Finished. All results will be saved to:")
logging.info(hypes['dirs']['output_dir'])
logging.info('Raw Results:')
utils.print_eval_dict(eval_dict, prefix='(raw) ')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/raw', summary_writer,
step)
logging.info('Smooth Results:')
names, res = zip(*eval_dict)
smoothed = py_smoother.update_weights(res)
eval_dict = zip(names, smoothed)
utils.print_eval_dict(eval_dict, prefix='(smooth)')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/smoothed',
summary_writer, step)
def do_analyze(logdir):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# prepaire the tv session
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
core.load_weights(logdir, sess, saver)
eval_dict, images = objective.tensor_eval(hypes, sess, image_pl,
softmax)
logging_file = os.path.join(logdir, "eval/analysis.log")
utils.create_filewrite_handler(logging_file)
utils.print_eval_dict(eval_dict)
_write_images_to_logdir(images, logdir)
return
def do_analyze(logdir):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# prepaire the tv session
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
core.load_weights(logdir, sess, saver)
eval_dict, images = objective.tensor_eval(hypes, sess, image_pl,
softmax)
logging_file = os.path.join(logdir, "eval/analysis.log")
utils.create_filewrite_handler(logging_file)
utils.print_eval_dict(eval_dict)
_write_images_to_logdir(images, logdir)
return
def run_united_evaluation(meta_hypes,
subhypes,
submodules,
subgraph,
tv_sess,
step=0):
logging.info('Running Evaluation Scripts.')
#Limit GPU usage when running on shared environment
summary_writer = tv_sess['writer']
models = meta_hypes['model_list']
sess = tv_sess['sess']
n = 0
for model in models:
eval_dict, images = submodules[model]['eval'].evaluate(
subhypes[model], sess, subgraph[model]['image_pl'],
subgraph[model]['inf_out'])
train._write_images_to_summary(images, summary_writer, step)
logging.info("%s Evaluation Finished. Results" % model)
logging.info('Raw Results:')
utils.print_eval_dict(eval_dict, prefix='(raw) ')
train._write_eval_dict_to_summary(eval_dict,
'Evaluation/%s/raw' % model,
summary_writer, step)
train._write_images_to_disk(meta_hypes, images, step)
logging.info("Evaluation Finished. All results will be saved to:")
logging.info(subhypes[model]['dirs']['output_dir'])
# Reset timer
start_time = time.time()
def run_training(hypes, modules, tv_graph, tv_sess, start_step=0):
"""Run one iteration of training."""
# Unpack operations for later use
summary = tf.Summary()
sess = tv_sess['sess']
summary_writer = tv_sess['writer']
solver = modules['solver']
display_iter = hypes['logging']['display_iter']
write_iter = hypes['logging'].get('write_iter', 5*display_iter)
eval_iter = hypes['logging']['eval_iter']
save_iter = hypes['logging']['save_iter']
image_iter = hypes['logging'].get('image_iter', 5*save_iter)
py_smoother = MedianSmoother(20)
dict_smoother = ExpoSmoother(0.95)
n = 0
eval_names, eval_ops = zip(*tv_graph['eval_list'])
# Run the training Step
start_time = time.time()
for step in xrange(start_step, hypes['solver']['max_steps']):
lr = solver.get_learning_rate(hypes, step)
feed_dict = {tv_graph['learning_rate']: lr}
if step % display_iter:
sess.run([tv_graph['train_op']], feed_dict=feed_dict)
# Write the summaries and print an overview fairly often.
elif step % display_iter == 0:
# Print status to stdout.
_, loss_value = sess.run([tv_graph['train_op'],
tv_graph['losses']['total_loss']],
feed_dict=feed_dict)
_print_training_status(hypes, step, loss_value, start_time, lr)
eval_results = sess.run(eval_ops, feed_dict=feed_dict)
_print_eval_dict(eval_names, eval_results, prefix=' (raw)')
dict_smoother.update_weights(eval_results)
smoothed_results = dict_smoother.get_weights()
_print_eval_dict(eval_names, smoothed_results, prefix='(smooth)')
# Reset timer
start_time = time.time()
if step % write_iter == 0:
# write values to summary
if FLAGS.summary:
summary_str = sess.run(tv_sess['summary_op'],
feed_dict=feed_dict)
summary_writer.add_summary(summary_str, global_step=step)
summary.value.add(tag='training/total_loss',
simple_value=float(loss_value))
summary.value.add(tag='training/learning_rate',
simple_value=lr)
summary_writer.add_summary(summary, step)
# Convert numpy types to simple types.
eval_results = np.array(eval_results)
eval_results = eval_results.tolist()
eval_dict = zip(eval_names, eval_results)
_write_eval_dict_to_summary(eval_dict, 'Eval/raw',
summary_writer, step)
eval_dict = zip(eval_names, smoothed_results)
_write_eval_dict_to_summary(eval_dict, 'Eval/smooth',
summary_writer, step)
# Do a evaluation and print the current state
if (step) % eval_iter == 0 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
# write checkpoint to disk
logging.info('Running Evaluation Script.')
eval_dict, images = modules['eval'].evaluate(
hypes, sess, tv_graph['image_pl'], tv_graph['inf_out'])
_write_images_to_summary(images, summary_writer, step)
logging.info("Evaluation Finished. All results will be saved to:")
logging.info(hypes['dirs']['output_dir'])
if images is not None and len(images) > 0:
name = str(n % 10) + '_' + images[0][0]
image_file = os.path.join(hypes['dirs']['image_dir'], name)
scp.misc.imsave(image_file, images[0][1])
n = n + 1
logging.info('Raw Results:')
utils.print_eval_dict(eval_dict, prefix='(raw) ')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/raw',
summary_writer, step)
logging.info('Smooth Results:')
names, res = zip(*eval_dict)
smoothed = py_smoother.update_weights(res)
eval_dict = zip(names, smoothed)
utils.print_eval_dict(eval_dict, prefix='(smooth)')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/smoothed',
summary_writer, step)
# Reset timer
start_time = time.time()
# Save a checkpoint periodically.
if (step) % save_iter == 0 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
# write checkpoint to disk
checkpoint_path = os.path.join(hypes['dirs']['output_dir'],
'model.ckpt')
tv_sess['saver'].save(sess, checkpoint_path, global_step=step)
# Reset timer
start_time = time.time()
if step % image_iter == 0 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
_write_images_to_disk(hypes, images, step)
def run_training(hypes, modules, tv_graph, tv_sess, start_step=0):
"""Run one iteration of training."""
# Unpack operations for later use
summary = tf.Summary()
sess = tv_sess['sess']
summary_writer = tv_sess['writer']
solver = modules['solver']
display_iter = hypes['logging']['display_iter']
write_iter = hypes['logging'].get('write_iter', 5*display_iter)
eval_iter = hypes['logging']['eval_iter']
save_iter = hypes['logging']['save_iter']
image_iter = hypes['logging'].get('image_iter', 5*save_iter)
py_smoother = MedianSmoother(20)
dict_smoother = ExpoSmoother(0.95)
n = 0
eval_names, eval_ops = zip(*tv_graph['eval_list'])
# Run the training Step
start_time = time.time()
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.7
for step in xrange(start_step, hypes['solver']['max_steps']):
logging.info("step %d", step)
lr = solver.get_learning_rate(hypes, step)
feed_dict = {tv_graph['learning_rate']: lr}
if step % display_iter:
sess.run([tv_graph['train_op']], feed_dict=feed_dict)
# Write the summaries and print an overview fairly often.
elif step % display_iter == 0:
# Print status to stdout.
_, loss_value = sess.run([tv_graph['train_op'],
tv_graph['losses']['total_loss']],
feed_dict=feed_dict)
_print_training_status(hypes, step, loss_value, start_time, lr)
eval_results = sess.run(eval_ops, feed_dict=feed_dict)
_print_eval_dict(eval_names, eval_results, prefix=' (raw)')
dict_smoother.update_weights(eval_results)
smoothed_results = dict_smoother.get_weights()
_print_eval_dict(eval_names, smoothed_results, prefix='(smooth)')
# Reset timer
start_time = time.time()
if step % write_iter == 3:
# write values to summary
if FLAGS.summary:
summary_str = sess.run(tv_sess['summary_op'],
feed_dict=feed_dict)
summary_writer.add_summary(summary_str, global_step=step)
summary.value.add(tag='training/total_loss',
simple_value=float(loss_value))
summary.value.add(tag='training/learning_rate',
simple_value=lr)
summary_writer.add_summary(summary, step)
# Convert numpy types to simple types.
eval_results = np.array(eval_results)
eval_results = eval_results.tolist()
eval_dict = zip(eval_names, eval_results)
_write_eval_dict_to_summary(eval_dict, 'Eval/raw',
summary_writer, step)
eval_dict = zip(eval_names, smoothed_results)
_write_eval_dict_to_summary(eval_dict, 'Eval/smooth',
summary_writer, step)
# Do a evaluation and print the current state
if (step) % eval_iter == 5 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
# write checkpoint to disk
logging.info('Running Evaluation Script.')
eval_dict, images = modules['eval'].evaluate(
hypes, sess, tv_graph['image_pl'], tv_graph['inf_out'])
_write_images_to_summary(images, summary_writer, step)
logging.info("Evaluation Finished. All results will be saved to:")
logging.info(hypes['dirs']['output_dir'])
if images is not None and len(images) > 0:
name = str(n % 10) + '_' + images[0][0]
image_file = os.path.join(hypes['dirs']['image_dir'], name)
scp.misc.imsave(image_file, images[0][1])
n = n + 1
logging.info('Raw Results:')
utils.print_eval_dict(eval_dict, prefix='(raw) ')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/raw',
summary_writer, step)
logging.info('Smooth Results:')
names, res = zip(*eval_dict)
smoothed = py_smoother.update_weights(res)
eval_dict = zip(names, smoothed)
utils.print_eval_dict(eval_dict, prefix='(smooth)')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/smoothed',
summary_writer, step)
# Reset timer
start_time = time.time()
# Save a checkpoint periodically.
if (step) % save_iter == 5 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
# write checkpoint to disk
checkpoint_path = os.path.join(hypes['dirs']['output_dir'],
'model.ckpt')
tv_sess['saver'].save(sess, checkpoint_path, global_step=step)
# Reset timer
start_time = time.time()
if step % image_iter == 10 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
_write_images_to_disk(hypes, images, step)
def run_united_training(meta_hypes, subhypes, submodules, subgraph, tv_sess,
start_step=0):
"""Run one iteration of training."""
# Unpack operations for later use
summary = tf.Summary()
#using the context manager launch the graph in session
sess = tv_sess['sess']
summary_writer = tv_sess['writer']
solvers = {}
for model in meta_hypes['models']:
solvers[model] = submodules[model]['solver']
#get the values from the hypes
display_iter = meta_hypes['logging']['display_iter']
write_iter = meta_hypes['logging'].get('write_iter', 5*display_iter)
eval_iter = meta_hypes['logging']['eval_iter']
save_iter = meta_hypes['logging']['save_iter']
image_iter = meta_hypes['logging'].get('image_iter', 5*save_iter)
models = meta_hypes['model_list']
#get the number of models
num_models = len(models)
py_smoothers = {}
dict_smoothers = {}
for model in models:
#smoothing pitch contours
py_smoothers[model] = train.MedianSmoother(5)
#exponetial smoothing to remove noises
dict_smoothers[model] = train.ExpoSmoother(0.95)
n = 0
eval_names = {}
eval_ops = {}
for model in models:
names, ops = zip(*subgraph[model]['eval_list'])
eval_names[model] = names
eval_ops[model] = ops
weights = meta_hypes['selection']['weights']
aweights = np.array([sum(weights[:i+1]) for i in range(len(weights))])
# eval_names, eval_ops = zip(*tv_graph['eval_list'])
# Run the training Step
start_time = time.time()
for step in xrange(start_step, meta_hypes['solver']['max_steps']):
# select on which model to run the training step
# select model randomly?
if not meta_hypes['selection']['random']:
if not meta_hypes['selection']['use_weights']:
# non-random selection
model = models[step % num_models]
else:
# non-random, some models are selected multiple times
select = np.argmax((aweights > step % aweights[-1]))
model = models[select]
else:
# random selection. Use weights
# to increase chance
r = random.random()
select = np.argmax((aweights > r))
model = models[select]
lr = solvers[model].get_learning_rate(subhypes[model], step)
feed_dict = {subgraph[model]['learning_rate']: lr}
sess.run([subgraph[model]['train_op']], feed_dict=feed_dict)
# Write the summaries and print an overview fairly often.
if step % display_iter == 0:
# Print status to stdout.
loss_values = {}
eval_results = {}
lrs = {}
if select == 1:
logging.info("Detection Loss was used.")
else:
logging.info("Segmentation Loss was used.")
for model in models:
loss_values[model] = sess.run(subgraph[model]['losses']
['total_loss'])
eval_results[model] = sess.run(eval_ops[model])
#update the weights of the model
dict_smoothers[model].update_weights(eval_results[model])
#get the learning rate for the iteration of the model
lrs[model] = solvers[model].get_learning_rate(subhypes[model],
step)
#call the function to print the training status
_print_training_status(meta_hypes, step,
loss_values,
start_time, lrs)
#print evaluation result for each of the models
for model in models:
train._print_eval_dict(eval_names[model], eval_results[model],
prefix=' (raw)')
smoothed_results = dict_smoothers[model].get_weights()
train._print_eval_dict(eval_names[model], smoothed_results,
prefix='(smooth)')
output = sess.run(subgraph['debug_ops'].values())
for name, res in zip(subgraph['debug_ops'].keys(), output):
logging.info("{} : {}".format(name, res))
if step % write_iter == 0:
# write values to summary
summary_str = sess.run(tv_sess['summary_op'],
feed_dict=feed_dict)
summary_writer.add_summary(summary_str,
global_step=step)
for model in models:
summary.value.add(tag='training/%s/total_loss' % model,
simple_value=float(loss_values[model]))
summary.value.add(tag='training/%s/learning_rate' % model,
simple_value=lrs[model])
summary_writer.add_summary(summary, step)
# Convert numpy types to simple types.
if False:
eval_results = np.array(eval_results)
eval_results = eval_results.tolist()
eval_dict = zip(eval_names[model], eval_results)
train._write_eval_dict_to_summary(eval_dict,
'Eval/%s/raw' % model,
summary_writer, step)
eval_dict = zip(eval_names[model], smoothed_results)
train._write_eval_dict_to_summary(eval_dict,
'Eval/%s/smooth' % model,
summary_writer, step)
# Reset timer
start_time = time.time()
# Do a evaluation and print the current state
if (step) % eval_iter == 0 and step > 0 or \
(step + 1) == meta_hypes['solver']['max_steps']:
# write checkpoint to disk
logging.info('Running Evaluation Scripts.')
for model in models:
eval_dict, images = submodules[model]['eval'].evaluate(
subhypes[model], sess,
subgraph[model]['image_pl'],
subgraph[model]['inf_out'])
train._write_images_to_summary(images, summary_writer, step)
if images is not None and len(images) > 0:
#get the name of the image
name = str(n % 10) + '_' + images[0][0]
#image directory
image_dir = subhypes[model]['dirs']['image_dir']
#image file name
image_file = os.path.join(image_dir, name)
#save the image
scp.misc.imsave(image_file, images[0][1])
n = n + 1
logging.info("%s Evaluation Finished. Results" % model)
logging.info('Raw Results:')
#print evaluation result
utils.print_eval_dict(eval_dict, prefix='(raw) ')
#write the evalutation summary
train._write_eval_dict_to_summary(
eval_dict, 'Evaluation/%s/raw' % model,
summary_writer, step)
logging.info('Smooth Results:')
names, res = zip(*eval_dict)
#smoothing image
smoothed = py_smoothers[model].update_weights(res)
eval_dict = zip(names, smoothed)
utils.print_eval_dict(eval_dict, prefix='(smooth)')
train._write_eval_dict_to_summary(
eval_dict, 'Evaluation/%s/smoothed' % model,
summary_writer, step)
if step % image_iter == 0 and step > 0 or \
(step + 1) == meta_hypes['solver']['max_steps']:
#save the image
train._write_images_to_disk(meta_hypes, images, step)
logging.info("Evaluation Finished. All results will be saved to:")
logging.info(subhypes[model]['dirs']['output_dir'])
# Reset timer
start_time = time.time()
# Save a checkpoint periodically.
if (step) % save_iter == 0 and step > 0 or \
(step + 1) == meta_hypes['solver']['max_steps']:
# write checkpoint to disk
checkpoint_path = os.path.join(meta_hypes['dirs']['output_dir'],
'model.ckpt')
tv_sess['saver'].save(sess, checkpoint_path, global_step=step)
# Reset timer
start_time = time.time()
return
def run_training(hypes, modules, tv_graph, tv_sess, start_step=0):
"""Run one iteration of training."""
# Unpack operations for later use
summary = tf.Summary()
sess = tv_sess['sess']
summary_writer = tv_sess['writer']
solver = modules['solver']
display_iter = hypes['logging']['display_iter']
write_iter = hypes['logging'].get('write_iter', 5*display_iter)
eval_iter = hypes['logging']['eval_iter']
save_iter = hypes['logging']['save_iter']
image_iter = hypes['logging'].get('image_iter', 5*save_iter)
dict_smoother = ExpoSmoother(0.95)
py_smoother = MedianSmoother(20)
n = 0
eval_names, eval_ops = zip(*tv_graph['eval_list'])
# Run the training Step
start_time = time.time()
for step in range(start_step, hypes['solver']['max_steps']):
regression_weights = solver.get_regression_weights(step, 1.0)
lr = solver.get_learning_rate(hypes, step)
feed_dict = {tv_graph['learning_rate']: lr,
hypes['solver']['regression_weights']: regression_weights}
# if step % display_iter:
# sess.run([tv_graph['train_op']], feed_dict=feed_dict)
# # Write the summaries and print an overview fairly often.
# elif step % display_iter == 0:
# # Print status to stdout.
# _, loss_value, training_loss, eval_results = sess.run([tv_graph['train_op'],
# tv_graph['losses']['total_loss'], tv_graph['losses'],
# eval_ops],
# feed_dict=feed_dict)
# _print_training_status(hypes, step, loss_value, start_time, lr)
# _print_eval_dict(eval_names, eval_results, prefix=' (raw)')
# dict_smoother.update_weights(eval_results)
# smoothed_results = dict_smoother.get_weights()
# _print_eval_dict(eval_names, smoothed_results, prefix='(smooth)')
# #logging.info('Regression Weights: Depth: %.2f, Location: %.2f, Corner: %.2f'%(regression_weights[0], \
# # regression_weights[1], regression_weights[2]))
# # Reset timer
# start_time = time.time()
# if step % write_iter == 0:
# # write values to summary
# if FLAGS.summary:
# summary_str = sess.run(tv_sess['summary_op'],
# feed_dict=feed_dict)
# summary_writer.add_summary(summary_str, global_step=step)
# summary.value.add(tag='training/total_loss',
# simple_value=float(loss_value))
# summary.value.add(tag='training/learning_rate',
# simple_value=lr)
# summary_writer.add_summary(summary, step)
# # Convert numpy types to simple types.
# eval_results = np.array(eval_results)
# eval_results = eval_results.tolist()
# eval_dict = zip(eval_names, eval_results)
# _write_eval_dict_to_summary(eval_dict, 'Eval/raw',
# summary_writer, step)
# eval_dict = zip(eval_names, smoothed_results)
# _write_eval_dict_to_summary(eval_dict, 'Eval/smooth',
# summary_writer, step)
# Do a evaluation and print the current state
if (step) % eval_iter == 0 \
or (step + 1) == hypes['solver']['max_steps']:
# write checkpoint to disk
logging.info('Running Evaluation Script.')
eval_dict, images = modules['eval'].evaluate(
hypes, sess, tv_graph['image_pl'], tv_graph['calib_pl'], tv_graph['xy_scale_pl'], tv_graph['inf_out'], tv_graph['encoder_out'])
_write_images_to_summary(images, summary_writer, step)
logging.info("Evaluation Finished. All results will be saved to:")
logging.info(hypes['dirs']['output_dir'])
if images is not None and len(images) > 0:
name = str(n % 10) + '_' + images[0][0]
image_file = os.path.join(hypes['dirs']['image_dir'], name)
scp.misc.imsave(image_file, images[0][1])
n = n + 1
logging.info('Raw Results:')
utils.print_eval_dict(eval_dict, prefix='(raw) ')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/raw',
summary_writer, step)
logging.info('Smooth Results:')
names, res = zip(*eval_dict)
smoothed = py_smoother.update_weights(res)
eval_dict = zip(names, smoothed)
utils.print_eval_dict(eval_dict, prefix='(smooth)')
_write_eval_dict_to_summary(eval_dict, 'Evaluation/smoothed',
summary_writer, step)
# Reset timer
start_time = time.time()
break
# Save a checkpoint periodically.
if (step) % save_iter == 0 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
# write checkpoint to disk
checkpoint_path = os.path.join(hypes['dirs']['output_dir'],
'model.ckpt')
tv_sess['saver'].save(sess, checkpoint_path, global_step=step)
# Reset timer
start_time = time.time()
if step % image_iter == 0 and step > 0 or \
(step + 1) == hypes['solver']['max_steps']:
_write_images_to_disk(hypes, images, step)
