def restore_map(self,
from_detection_checkpoint=True,
restore_box_predictor=False):
"""Returns a map of variables to load from a foreign checkpoint.
See parent class for details.
Args:
from_detection_checkpoint: whether to restore from a full detection
checkpoint (with compatible variable names) or to restore from a
classification checkpoint for initialization prior to training.
restore_box_predictor: Whether to restore the weights of box predictor.
Returns:
A dict mapping variable names (to load from a checkpoint) to variables in
the model graph.
"""
variables_to_restore = {}
for variable in tf.global_variables():
var_name = variable.op.name
if variable.op.name.startswith(self._extract_features_scope):
if not from_detection_checkpoint:
var_name = (re.split(
'^' + self._extract_features_scope + '/',
var_name)[-1])
log.infov(' Restore [%s]', var_name)
variables_to_restore[var_name] = variable
elif var_name.startswith(self._box_predictor_scope):
if not from_detection_checkpoint:
var_name = (re.split('^' + self._box_predictor_scope + '/',
var_name)[-1])
if restore_box_predictor:
log.infov(' Restore [%s]', var_name)
variables_to_restore[var_name] = variable
else:
log.warn(' Skip [%s]', var_name)
continue
return variables_to_restore
def build(init_file):
"""Build initializers based on the init file"""
initial_values = joblib.load(init_file)
initializers = {}
for name, values_dict in initial_values.iteritems():
log.infov('Build initializer for layer [%s].', name)
initializers[name] = build_layer_initializers(values_dict)
return initializers
def write_metrics(metrics, global_step, summary_dir, eval_config=None):
"""Write metrics to a summary directory.
Args:
metrics: A dictionary containing metric names and values.
global_step: Global step at which the metrics are computed.
summary_dir: Directory to write tensorflow summaries to.
"""
log.infov('Writing metrics to tf summary.')
summary_writer = tf.summary.FileWriter(summary_dir)
if eval_config is not None:
str_list = get_string_list_for_nms(eval_config.nms_type,
eval_config.nms_threshold,
eval_config.soft_nms_sigma)
nms_str = ' / '.join(str_list).upper()
log.warn('[Post Process: %s]', nms_str)
for key in sorted(metrics):
summary = tf.Summary(value=[
tf.Summary.Value(tag=key, simple_value=metrics[key]),
])
summary_writer.add_summary(summary, global_step)
log.info('%s: %f', key, metrics[key])
summary_writer.close()
log.infov('Metrics written to tf summary.')
def train(train_op,
logdir,
train_step_fn=train_step,
train_step_kwargs=_USE_DEFAULT,
log_every_n_steps=1,
graph=None,
master='',
is_chief=True,
global_step=None,
number_of_steps=None,
init_op=_USE_DEFAULT,
init_feed_dict=None,
local_init_op=_USE_DEFAULT,
init_fn=None,
ready_op=_USE_DEFAULT,
summary_op=_USE_DEFAULT,
save_summaries_secs=600,
summary_writer=_USE_DEFAULT,
startup_delay_steps=0,
saver=None,
save_interval_secs=600,
sync_optimizer=None,
session_config=None,
session_wrapper=None,
trace_every_n_steps=None,
batch_size=1,
num_examples=None,
config_summary_list=None):
"""Runs a training loop using a TensorFlow supervisor.
When the sync_optimizer is supplied, gradient updates are applied
synchronously. Otherwise, gradient updates are applied asynchronous.
Args:
train_op: A `Tensor` that, when executed, will apply the gradients and
return the loss value.
logdir: The directory where training logs are written to. If None, model
checkpoints and summaries will not be written.
train_step_fn: The function to call in order to execute a single gradient
step. The function must have take exactly four arguments: the current
session, the `train_op` `Tensor`, a global step `Tensor` and a dictionary.
train_step_kwargs: A dictionary which is passed to the `train_step_fn`. By
default, two `Boolean`, scalar ops called "should_stop" and "should_log"
are provided.
log_every_n_steps: The frequency, in terms of global steps, that the loss
and global step and logged.
graph: The graph to pass to the supervisor. If no graph is supplied the
default graph is used.
master: The address of the tensorflow master.
is_chief: Specifies whether or not the training is being run by the primary
replica during replica training.
global_step: The `Tensor` representing the global step. If left as `None`,
then slim.variables.get_or_create_global_step() is used.
number_of_steps: The max number of gradient steps to take during training,
as measured by 'global_step': training will stop if global_step is
greater than 'number_of_steps'. If the value is left as None, training
proceeds indefinitely.
init_op: The initialization operation. If left to its default value, then
the session is initialized by calling `tf.global_variables_initializer()`.
init_feed_dict: A feed dictionary to use when executing the `init_op`.
local_init_op: The local initialization operation. If left to its default
value, then the session is initialized by calling
`tf.local_variables_initializer()` and `tf.tables_initializer()`.
init_fn: An optional callable to be executed after `init_op` is called. The
callable must accept one argument, the session being initialized.
ready_op: Operation to check if the model is ready to use. If left to its
default value, then the session checks for readiness by calling
`tf.report_uninitialized_variables()`.
summary_op: The summary operation.
save_summaries_secs: How often, in seconds, to save summaries.
summary_writer: `SummaryWriter` to use. Can be `None`
to indicate that no summaries should be written. If unset, we
create a SummaryWriter.
startup_delay_steps: The number of steps to wait for before beginning. Note
that this must be 0 if a sync_optimizer is supplied.
saver: Saver to save checkpoints. If None, a default one will be created
and used.
save_interval_secs: How often, in seconds, to save the model to `logdir`.
sync_optimizer: an instance of tf.train.SyncReplicasOptimizer, or a list of
them. If the argument is supplied, gradient updates will be synchronous.
If left as `None`, gradient updates will be asynchronous.
session_config: An instance of `tf.ConfigProto` that will be used to
configure the `Session`. If left as `None`, the default will be used.
session_wrapper: A function that takes a `tf.Session` object as the only
argument and returns a wrapped session object that has the same methods
that the original object has, or `None`. Iff not `None`, the wrapped
object will be used for training.
trace_every_n_steps: produce and save a `Timeline` in Chrome trace format
and add it to the summaries every `trace_every_n_steps`. If None, no trace
information will be produced or saved.
batch_size: batch size.
num_examples: The number of examples in dataset for training.
dubug_tensors: Additional tensors to run for debugging.
Returns:
the value of the loss function after training.
Raises:
ValueError: if `train_op` is empty or if `startup_delay_steps` is
non-zero when `sync_optimizer` is supplied, if `number_of_steps` is
negative, or if `trace_every_n_steps` is not `None` and no `logdir` is
provided.
"""
if train_op is None:
raise ValueError('train_op cannot be None.')
if not isinstance(train_op, list):
train_op = [train_op]
# Allocate log function to each step.
log_fn_list = [log.info, log.infov]
def _iter_log_fn():
for log_fn in log_fn_list:
yield log_fn
it = itertools.cycle(_iter_log_fn())
current_log_fn = it.next()
if logdir is None:
if summary_op != _USE_DEFAULT:
raise ValueError('Cannot provide summary_op because logdir=None')
if saver is not None:
raise ValueError('Cannot provide saver because logdir=None')
if trace_every_n_steps is not None:
raise ValueError('Cannot provide trace_every_n_steps because '
'logdir=None')
if isinstance(sync_optimizer,
sync_replicas_optimizer.SyncReplicasOptimizer):
sync_optimizer = [sync_optimizer]
if sync_optimizer is not None and startup_delay_steps > 0:
raise ValueError(
'startup_delay_steps must be zero when sync_optimizer is supplied.'
)
if number_of_steps is not None and number_of_steps <= 0:
raise ValueError(
'`number_of_steps` must be either None or a positive number.')
graph = graph or ops.get_default_graph()
with graph.as_default():
if global_step is None:
global_step = training_util.get_or_create_global_step()
saver = saver or tf_saver.Saver()
if sync_optimizer is not None:
for opt in sync_optimizer:
if not isinstance(
opt, sync_replicas_optimizer.SyncReplicasOptimizer):
raise ValueError(
'`sync_optimizer` must be a tf.train.SyncReplicasOptimizer.'
)
with ops.name_scope('init_ops'):
if init_op == _USE_DEFAULT:
init_op = variables.global_variables_initializer()
if ready_op == _USE_DEFAULT:
ready_op = variables.report_uninitialized_variables()
if local_init_op == _USE_DEFAULT:
local_init_op = control_flow_ops.group(
variables.local_variables_initializer(),
lookup_ops.tables_initializer())
if sync_optimizer is not None and isinstance(sync_optimizer, list):
with ops.control_dependencies(
[local_init_op] if local_init_op is not None else []):
if is_chief:
local_init_op = control_flow_ops.group(
*[opt.chief_init_op for opt in sync_optimizer])
else:
local_init_op = control_flow_ops.group(
*
[opt.local_step_init_op for opt in sync_optimizer])
ready_for_local_init_op = control_flow_ops.group(
*[opt.ready_for_local_init_op for opt in sync_optimizer])
else:
ready_for_local_init_op = None
if summary_op == _USE_DEFAULT:
summary_op = summary.merge_all()
if summary_writer == _USE_DEFAULT:
summary_writer = supervisor.Supervisor.USE_DEFAULT
if is_chief and sync_optimizer is not None:
# Need to create these BEFORE the supervisor finalizes the graph:
init_tokens_op = [
opt.get_init_tokens_op() for opt in sync_optimizer
]
chief_queue_runner = [
opt.get_chief_queue_runner() for opt in sync_optimizer
]
if train_step_kwargs == _USE_DEFAULT:
with ops.name_scope('train_step'):
train_step_kwargs = {}
if number_of_steps:
should_stop_op = math_ops.greater_equal(
global_step, number_of_steps)
else:
should_stop_op = constant_op.constant(False)
train_step_kwargs['should_stop'] = should_stop_op
if log_every_n_steps > 0:
train_step_kwargs['should_log'] = math_ops.equal(
math_ops.mod(global_step, log_every_n_steps), 0)
if is_chief and trace_every_n_steps is not None:
train_step_kwargs['should_trace'] = math_ops.equal(
math_ops.mod(global_step, trace_every_n_steps), 0)
train_step_kwargs['logdir'] = logdir
sv = supervisor.Supervisor(graph=graph,
is_chief=is_chief,
logdir=logdir,
init_op=init_op,
init_feed_dict=init_feed_dict,
local_init_op=local_init_op,
ready_for_local_init_op=ready_for_local_init_op,
ready_op=ready_op,
summary_op=summary_op,
summary_writer=summary_writer,
global_step=global_step,
saver=saver,
save_summaries_secs=save_summaries_secs,
save_model_secs=save_interval_secs,
init_fn=init_fn)
if summary_writer is not None:
train_step_kwargs['summary_writer'] = sv.summary_writer
steps_in_epoch = int(num_examples / batch_size)
total_loss = 0.0
should_retry = True
while should_retry:
try:
should_retry = False
with sv.managed_session(master,
start_standard_services=False,
config=session_config) as sess:
log.infov('Starting Session.')
if session_wrapper is not None:
log.info('Wrapping session with wrapper function: %s',
session_wrapper)
sess = session_wrapper(sess)
if is_chief:
if logdir:
sv.start_standard_services(sess)
elif startup_delay_steps > 0:
_wait_for_step(
sess, global_step,
min(startup_delay_steps, number_of_steps
or sys.maxint))
threads = sv.start_queue_runners(sess)
log.infov('Starting Queues.')
if is_chief and sync_optimizer is not None:
sv.start_queue_runners(sess, chief_queue_runner)
sess.run(init_tokens_op)
sess.graph.finalize()
# try:
if config_summary_list is not None:
for config_summary in config_summary_list:
sv.summary_writer.add_summary(
config_summary.eval(session=sess))
while not sv.should_stop():
for _train_op in train_op:
total_loss, should_stop, np_global_step = train_step_fn(
sess, _train_op, global_step, train_step_kwargs,
batch_size, steps_in_epoch, current_log_fn)
if should_stop:
log.infov('Stopping Training.')
sv.request_stop()
break
# except errors.OutOfRangeError:
# # OutOfRangeError is thrown when epoch limit per
# # tf.train.limit_epochs is reached.
# log.warn('Caught OutOfRangeError. Stopping Training.')
if logdir and sv.is_chief:
log.warn('Finished training! Saving model to disk.')
sv.saver.save(sess,
sv.save_path,
global_step=sv.global_step)
sv.stop(threads, close_summary_writer=True)
def _last_checkpoint_path(sv_save_path,
additional_dir_name='last'):
dir_list = sv_save_path.split('/')
dir_list.insert(-1, 'last')
last_checkpoint_dir_path = '/'.join(dir_list[:-1])
last_checkpoint_path = '/'.join(dir_list)
return last_checkpoint_dir_path, last_checkpoint_path
# Save the last checkpoint again to a 'last' directory for the next training with
# different configuration.
last_checkpoint_dir_path, last_checkpoint_path = _last_checkpoint_path(
sv.save_path, 'last')
if os.path.exists(last_checkpoint_dir_path):
shutil.rmtree(last_checkpoint_dir_path)
os.makedirs(last_checkpoint_dir_path)
sv.saver.save(sess,
last_checkpoint_path,
global_step=sv.global_step)
except errors.AbortedError:
# Always re-run on AbortedError as it indicates a restart of one of the
# distributed tensorflow servers.
log.warn('Retrying training!')
should_retry = True
return total_loss
def train_step(sess, train_op, global_step, train_step_kwargs, batch_size,
steps_in_epoch, log_fn):
"""Function that takes a gradient step and specifies whether to stop.
Args:
sess: The current session.
train_op: An `Operation` that evaluates the gradients and returns the
total loss.
global_step: A `Tensor` representing the global training step.
train_step_kwargs: A dictionary of keyword arguments.
batch_size: batch size.
steps_in_epoch: The number of steps in one epoch.
Returns:
The total loss and a boolean indicating whether or not to stop training.
Raises:
ValueError: if 'should_trace' is in `train_step_kwargs` but `logdir` is not.
"""
start_time = time.time()
trace_run_options = None
run_metadata = None
if 'should_trace' in train_step_kwargs:
if 'logdir' not in train_step_kwargs:
raise ValueError(
'logdir must be present in train_step_kwargs when '
'should_trace is present')
if sess.run(train_step_kwargs['should_trace']):
trace_run_options = config_pb2.RunOptions(
trace_level=config_pb2.RunOptions.FULL_TRACE)
run_metadata = config_pb2.RunMetadata()
out = sess.run([train_op, global_step],
options=trace_run_options,
run_metadata=run_metadata)
total_loss, np_global_step = out[:2]
time_elapsed = time.time() - start_time
if run_metadata is not None:
tl = timeline.Timeline(run_metadata.step_stats)
trace = tl.generate_chrome_trace_format()
trace_filename = os.path.join(train_step_kwargs['logdir'],
'tf_trace-%d.json' % np_global_step)
log.infov('Writing trace to %s', trace_filename)
file_io.write_string_to_file(trace_filename, trace)
if 'summary_writer' in train_step_kwargs:
train_step_kwargs['summary_writer'].add_run_metadata(
run_metadata, 'run_metadata-%d' % np_global_step)
if 'should_log' in train_step_kwargs:
if sess.run(train_step_kwargs['should_log']):
log_fn((
"[train step {step:4d} / epoch {epoch:.2f}] " +
"loss: {loss:.5f} " +
"({sec_per_batch:.3f} sec/batch, {images_per_sec:.3f} instances/sec)"
).format(epoch=float(np_global_step) / steps_in_epoch,
step=np_global_step,
loss=total_loss,
sec_per_batch=time_elapsed,
images_per_sec=batch_size / time_elapsed))
# TODO(nsilberman): figure out why we can't put this into sess.run. The
# issue right now is that the stop check depends on the global step. The
# increment of global step often happens via the train op, which used
# created using optimizer.apply_gradients.
#
# Since running `train_op` causes the global step to be incremented, one
# would expected that using a control dependency would allow the
# should_stop check to be run in the same session.run call:
#
# with ops.control_dependencies([train_op]):
# should_stop_op = ...
#
# However, this actually seems not to work on certain platforms.
if 'should_stop' in train_step_kwargs:
should_stop = sess.run(train_step_kwargs['should_stop'])
else:
should_stop = False
return total_loss, should_stop, np_global_step
def run_checkpoint_once(tensor_dict,
update_op,
summary_dir,
aggregated_result_processor=None,
batch_processor=None,
checkpoint_dirs=None,
variables_to_restore=None,
restore_fn=None,
num_batches=1,
master='',
save_graph=False,
save_graph_dir='',
metric_names_to_values=None,
keys_to_exclude_from_results=(),
eval_config=None,
gpu_fraction=0.0,
categories=[]):
"""Evaluates both python metrics and tensorflow slim metrics.
Python metrics are processed in batch by the aggregated_result_processor,
while tensorflow slim metrics statistics are computed by running
metric_names_to_updates tensors and aggregated using metric_names_to_values
tensor.
Args:
tensor_dict: a dictionary holding tensors representing a batch of detections
and corresponding groundtruth annotations.
update_op: a tensorflow update op that will run for each batch along with
the tensors in tensor_dict..
summary_dir: a directory to write metrics summaries.
aggregated_result_processor: a function taking one arguments:
1. result_lists: a dictionary with keys matching those in tensor_dict
and corresponding values being the list of results for each tensor
in tensor_dict. The length of each such list is num_batches.
batch_processor: a function taking four arguments:
1. tensor_dict: the same tensor_dict that is passed in as the first
argument to this function.
2. sess: a tensorflow session
3. batch_index: an integer representing the index of the batch amongst
all batches
4. update_op: a tensorflow update op that will run for each batch.
and returns result_dict, a dictionary of results for that batch.
By default, batch_processor is None, which defaults to running:
return sess.run(tensor_dict)
To skip an image, it suffices to return an empty dictionary in place of
result_dict.
checkpoint_dirs: list of directories to load into an EnsembleModel. If it
has only one directory, EnsembleModel will not be used -- a DetectionModel
will be instantiated directly. Not used if restore_fn is set.
variables_to_restore: None, or a dictionary mapping variable names found in
a checkpoint to model variables. The dictionary would normally be
generated by creating a tf.train.ExponentialMovingAverage object and
calling its variables_to_restore() method. Not used if restore_fn is set.
restore_fn: None, or a function that takes a tf.Session object and correctly
restores all necessary variables from the correct checkpoint file. If
None, attempts to restore from the first directory in checkpoint_dirs.
num_batches: the number of batches to use for evaluation.
master: the location of the Tensorflow session.
save_graph: whether or not the Tensorflow graph is stored as a pbtxt file.
save_graph_dir: where to store the Tensorflow graph on disk. If save_graph
is True this must be non-empty.
metric_names_to_values: A dictionary containing metric names to tensors
which will be evaluated after processing all batches
of [tensor_dict, update_op]. If any metrics depend on statistics computed
during each batch ensure that `update_op` tensor has a control dependency
on the update ops that compute the statistics.
keys_to_exclude_from_results: keys in tensor_dict that will be excluded
from results_list. Note that the tensors corresponding to these keys will
still be evaluated for each batch, but won't be added to results_list.
gpu_fraction: GPU memory fraction for evaluation.
Raises:
ValueError: if restore_fn is None and checkpoint_dirs doesn't have at least
one element.
ValueError: if save_graph is True and save_graph_dir is not defined.
"""
if save_graph and not save_graph_dir:
raise ValueError('`save_graph_dir` must be defined.')
tf_config = tf.ConfigProto()
if gpu_fraction == 0.0:
tf_config.gpu_options.allow_growth = True
else:
tf_config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction
sess = tf.Session(master, graph=tf.get_default_graph(), config=tf_config)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
if restore_fn:
checkpoint_file = restore_fn(sess)
else:
if not checkpoint_dirs:
raise ValueError('`checkpoint_dirs` must have at least one entry.')
checkpoint_file = tf.train.latest_checkpoint(checkpoint_dirs[0])
saver = tf.train.Saver(variables_to_restore)
saver.restore(sess, checkpoint_file)
if save_graph:
tf.train.write_graph(sess.graph_def, save_graph_dir, 'eval.pbtxt')
valid_keys = list(
set(tensor_dict.keys()) - set(keys_to_exclude_from_results))
if eval_config.submission_format_output:
for key in tensor_dict.keys():
if not key in valid_keys:
del tensor_dict[key]
result_lists = {key: [] for key in valid_keys}
counters = {'skipped': 0, 'success': 0}
other_metrics = None
with tf.contrib.slim.queues.QueueRunners(sess):
try:
for batch in range(int(num_batches)):
if (batch + 1) % 100 == 0:
log.info('Running eval ops batch %d/%d', batch + 1,
num_batches)
if not batch_processor:
try:
(result_dict, _) = sess.run([tensor_dict, update_op])
counters['success'] += 1
except tf.errors.InvalidArgumentError:
log.warn('Skipping image')
counters['skipped'] += 1
result_dict = {}
else:
result_dict = batch_processor(tensor_dict, sess, batch,
counters, update_op)
for key in result_dict:
if key in valid_keys:
result_lists[key].append(result_dict[key])
log.info('Running eval ops batch %d/%d', batch + 1,
num_batches) # print final loop also
if metric_names_to_values is not None:
other_metrics = sess.run(metric_names_to_values)
log.infov('Running eval batches done.')
except tf.errors.OutOfRangeError:
log.warn('Done evaluating -- epoch limit reached')
finally:
# Save raw detection results
detection_results = {}
for idx, image_id in enumerate(result_lists['image_id']):
box = result_lists['detection_boxes'][idx]
converted_boxes = np.column_stack([
(box[:, 3] + box[:, 1]) / 2, # cx
(box[:, 2] + box[:, 0]) / 2, # cy
box[:, 3] - box[:, 1], # w
box[:, 2] - box[:, 0], # h
])
det = np.hstack([
converted_boxes,
np.expand_dims(result_lists['detection_scores'][idx], 1)
])
detection_results[image_id.split('/')[-1].split('.')[0]] \
= np.array(sorted(det, key=lambda x: x[4]))
result_path = os.path.join(summary_dir, 'detection_results')
if not os.path.exists(result_path):
os.makedirs(result_path)
# Save detection results for submission
if eval_config.submission_format_output:
save_detection_results_for_submission(result_lists, categories,
summary_dir,
eval_config.metrics_set)
else:
# checkpoint_path = tf.train.latest_checkpoint(checkpoint_dirs[0])
last_step = checkpoint_file.split('-')[-1]
if last_step.find('/') != -1:
last_step = '0'
# result_path = os.path.join(result_path, 'detection_result_' + last_step)
# with open(result_path, 'w') as f:
# pickle.dump(detection_results, f)
# log.info('Detection results saved to %s' % result_path)
# evaluate results
metrics = aggregated_result_processor(result_lists, sess)
mtl_metrics = mtl_util.get_mtl_metrics(result_lists)
metrics.update(mtl_metrics)
if other_metrics is not None:
metrics.update(other_metrics)
# Loss
if eval_config.calc_loss:
losses = aggregated_loss(result_lists)
metrics.update(losses)
global_step = tf.train.global_step(sess,
slim.get_global_step())
write_metrics(metrics, global_step, summary_dir, eval_config)
# save best ckpt
save_best_ckpt(sess, metrics, variables_to_restore,
checkpoint_file, eval_config)
log.warn('# success: %d', counters['success'])
log.warn('# skipped: %d', counters['skipped'])
sess.close()
def visualize_detection_results(result_dict,
tag,
global_step,
categories,
summary_dir='',
export_dir='',
agnostic_mode=False,
show_groundtruth=False,
min_score_thresh=.5,
max_num_predictions=20):
"""Visualizes detection results and writes visualizations to image summaries.
This function visualizes an image with its detected bounding boxes and writes
to image summaries which can be viewed on tensorboard. It optionally also
writes images to a directory. In the case of missing entry in the label map,
unknown class name in the visualization is shown as "N/A".
Args:
result_dict: a dictionary holding groundtruth and detection
data corresponding to each image being evaluated. The following keys
are required:
'original_image': a numpy array representing the image with shape
[1, height, width, 3]
'detection_boxes': a numpy array of shape [N, 4]
'detection_scores': a numpy array of shape [N]
'detection_classes': a numpy array of shape [N]
The following keys are optional:
'groundtruth_boxes': a numpy array of shape [N, 4]
'groundtruth_keypoints': a numpy array of shape [N, num_keypoints, 2]
Detections are assumed to be provided in decreasing order of score and for
display, and we assume that scores are probabilities between 0 and 1.
tag: tensorboard tag (string) to associate with image.
global_step: global step at which the visualization are generated.
categories: a list of dictionaries representing all possible categories.
Each dict in this list has the following keys:
'id': (required) an integer id uniquely identifying this category
'name': (required) string representing category name
e.g., 'cat', 'dog', 'pizza'
'supercategory': (optional) string representing the supercategory
e.g., 'animal', 'vehicle', 'food', etc
summary_dir: the output directory to which the image summaries are written.
export_dir: the output directory to which images are written. If this is
empty (default), then images are not exported.
agnostic_mode: boolean (default: False) controlling whether to evaluate in
class-agnostic mode or not.
show_groundtruth: boolean (default: False) controlling whether to show
groundtruth boxes in addition to detected boxes
min_score_thresh: minimum score threshold for a box to be visualized
max_num_predictions: maximum number of detections to visualize
Raises:
ValueError: if result_dict does not contain the expected keys (i.e.,
'original_image', 'detection_boxes', 'detection_scores',
'detection_classes')
"""
if not set([
'original_image', 'detection_boxes', 'detection_scores',
'detection_classes'
]).issubset(set(result_dict.keys())):
raise ValueError('result_dict does not contain all expected keys.')
if show_groundtruth and 'groundtruth_boxes' not in result_dict:
raise ValueError(
'If show_groundtruth is enabled, result_dict must contain '
'groundtruth_boxes.')
log.infov('Creating detection visualizations.')
category_index = label_map_util.create_category_index(categories)
image = np.squeeze(result_dict['original_image'], axis=0)
detection_boxes = result_dict['detection_boxes']
detection_scores = result_dict['detection_scores']
detection_classes = np.int32((result_dict['detection_classes']))
detection_keypoints = result_dict.get('detection_keypoints', None)
detection_masks = result_dict.get('detection_masks', None)
# Plot groundtruth underneath detections
if show_groundtruth:
groundtruth_boxes = result_dict['groundtruth_boxes']
boxes_area = (groundtruth_boxes[:, 2] - groundtruth_boxes[:, 0]) * (
groundtruth_boxes[:, 3] - groundtruth_boxes[:, 1])
area_index_total = np.argsort(boxes_area) # ascending order
groundtruth_boxes_ordered = np.array(
[groundtruth_boxes[i] for i in area_index_total])
groundtruth_keypoints = result_dict.get('groundtruth_keypoints', None)
vis_utils.visualize_boxes_and_labels_on_image_array(
image,
groundtruth_boxes_ordered,
None,
None,
category_index,
keypoints=groundtruth_keypoints,
use_normalized_coordinates=False,
max_boxes_to_draw=None)
vis_utils.visualize_boxes_and_labels_on_image_array(
image,
detection_boxes,
detection_classes,
detection_scores,
category_index,
instance_masks=detection_masks,
keypoints=detection_keypoints,
use_normalized_coordinates=False,
max_boxes_to_draw=max_num_predictions,
min_score_thresh=min_score_thresh,
agnostic_mode=agnostic_mode)
if export_dir:
export_path = os.path.join(export_dir, 'export-{}.png'.format(tag))
vis_utils.save_image_array_as_png(image, export_path)
summary_value = [
tf.Summary.Value(
tag=tag + '/image',
image=tf.Summary.Image(encoded_image_string=vis_utils.
encode_image_array_as_png_str(image)))
]
summary = tf.Summary(value=summary_value)
summary_writer = tf.summary.FileWriter(summary_dir)
summary_writer.add_summary(summary, global_step)
summary_writer.close()
time.sleep(1)
log.warn('Detection visualizations written to summary with tag %s.', tag)
def evaluate_detection_results_coco(result_lists,
categories,
label_id_offset=1,
iou_thres=0.5,
corloc_summary=False,
nms_type='standard',
nms_thres=1.0,
soft_nms_sigma=0.5,
global_step=0,
eval_dir='',
eval_config=None):
"""Computes MS-COCO detection metrics given groundtruth and detections.
This function computes MS-COCO metrics. This function by default
takes detections and groundtruth boxes encoded in result_lists and writes
evaluation results to tf summaries which can be viewed on tensorboard.
Args:
result_lists: a dictionary holding lists of groundtruth and detection
data corresponding to each image being evaluated. The following keys
are required:
'image_id': a list of string ids
'detection_boxes': a list of float32 numpy arrays of shape [N, 4]
'detection_scores': a list of float32 numpy arrays of shape [N]
'detection_classes': a list of int32 numpy arrays of shape [N]
'groundtruth_boxes': a list of float32 numpy arrays of shape [M, 4]
'groundtruth_classes': a list of int32 numpy arrays of shape [M]
and the remaining fields below are optional:
'difficult': a list of boolean arrays of shape [M] indicating the
difficulty of groundtruth boxes. Some datasets like PASCAL VOC provide
this information and it is used to remove difficult examples from eval
in order to not penalize the models on them.
Note that it is okay to have additional fields in result_lists --- they
are simply ignored.
categories: a list of dictionaries representing all possible categories.
Each dict in this list has the following keys:
'id': (required) an integer id uniquely identifying this category
'name': (required) string representing category name
e.g., 'cat', 'dog', 'pizza'
label_id_offset: an integer offset for the label space.
iou_thres: float determining the IoU threshold at which a box is considered
correct. Defaults to the standard 0.5.
corloc_summary: boolean. If True, also outputs CorLoc metrics.
nms_type: type of NMS (standard|soft-linear|soft_gaussian)
nms_thres: iou threshold for non maximum suppression.
soft_nms_sigma: Soft NMS sigma.
Returns:
A dictionary of metric names to scalar values.
Raises:
ValueError: if the set of keys in result_lists is not a superset of the
expected list of keys. Unexpected keys are ignored.
ValueError: if the lists in result_lists have inconsistent sizes.
"""
# check for expected keys in result_lists
expected_keys = [
'detection_boxes', 'detection_scores', 'detection_classes', 'image_id'
]
expected_keys += ['groundtruth_boxes', 'groundtruth_classes']
if not set(expected_keys).issubset(set(result_lists.keys())):
raise ValueError('result_lists does not have expected key set.')
num_results = len(result_lists[expected_keys[0]])
for key in expected_keys:
if len(result_lists[key]) != num_results:
raise ValueError('Inconsistent list sizes in result_lists')
categories = copy.deepcopy(categories)
for idx in range(len(categories)):
categories[idx]['id'] -= label_id_offset
# num_classes (maybe encoded as categories)
num_classes = max([cat['id'] for cat in categories]) + 1
log.infov('Computing COCO metrics on results.')
if all(image_id.isdigit() for image_id in result_lists['image_id']):
image_ids = [int(image_id) for image_id in result_lists['image_id']]
else:
image_ids = range(num_results)
evaluator = object_detection_evaluation.CocoEvaluation(
num_classes,
matching_iou_threshold=iou_thres,
nms_type=nms_type,
nms_iou_threshold=nms_thres,
soft_nms_sigma=soft_nms_sigma)
for idx, image_id in enumerate(image_ids):
evaluator.add_single_ground_truth_image_info(
image_id, result_lists['groundtruth_boxes'][idx],
result_lists['groundtruth_classes'][idx] - label_id_offset)
evaluator.add_single_detected_image_info(
image_id, result_lists['detection_boxes'][idx],
result_lists['detection_scores'][idx],
result_lists['detection_classes'][idx] - label_id_offset)
metric_names = [
'AP', 'AP_IoU50', 'AP_IoU75', 'AP_small', 'AP_medium', 'AP_large',
'AR_max1', 'AR_max10', 'AR_max100', 'AR_small', 'AR_medium', 'AR_large'
]
metrics = dict()
if eval_config is None:
eval_metric_index = [0]
eval_class_type = 1
eval_ann_filename = '../data/mscoco/annotations/instances_eval2014.json'
else:
eval_metric_index = eval_config.coco_eval_options.eval_metric_index
eval_class_type = eval_config.coco_eval_options.eval_class_type
eval_ann_filename = eval_config.coco_eval_options.eval_ann_filename
if len(eval_metric_index) == 0:
eval_metric_index = [0]
if eval_ann_filename == '':
eval_ann_filename = '../data/mscoco/annotations/instances_eval2014.json'
if min(eval_metric_index) < 0 or max(eval_metric_index) > 11:
raise ValueError('eval_metric_index')
return metrics
if eval_class_type < 0 or eval_class_type > 1:
raise ValueError('eval_class_type')
return metrics
if eval_class_type == 0:
eval_cat_index = [0]
else:
eval_cat_index = range(num_classes + 1)
coco_metrics = evaluator.evaluate(
eval_cat_index,
eval_ann_filename) # coco_metrics[num_classes+1(all)][12]
if coco_metrics is None:
return metrics
category_index = label_map_util.create_category_index(categories)
for cat_id in range(num_classes + 1):
if cat_id == 0:
class_name = 'All'
elif cat_id - 1 in category_index.keys():
class_name = category_index[cat_id - 1]['name']
else:
continue
if not cat_id in eval_cat_index:
continue
for metric_id, metric_name in enumerate(metric_names):
if not metric_id in eval_metric_index:
continue
display_name = 'COCO_Eval/%s/%s' % (class_name, metric_name)
metrics[display_name] = coco_metrics[cat_id][metric_id]
return metrics
def evaluate_detection_results_pascal_voc(result_lists,
categories,
label_id_offset=1,
iou_thres=0.5,
corloc_summary=False,
nms_type='standard',
nms_thres=1.0,
soft_nms_sigma=0.5,
global_step=0,
eval_dir='',
eval_config=None):
"""Computes Pascal VOC detection metrics given groundtruth and detections.
This function computes Pascal VOC metrics. This function by default
takes detections and groundtruth boxes encoded in result_lists and writes
evaluation results to tf summaries which can be viewed on tensorboard.
Args:
result_lists: a dictionary holding lists of groundtruth and detection
data corresponding to each image being evaluated. The following keys
are required:
'image_id': a list of string ids
'detection_boxes': a list of float32 numpy arrays of shape [N, 4]
'detection_scores': a list of float32 numpy arrays of shape [N]
'detection_classes': a list of int32 numpy arrays of shape [N]
'groundtruth_boxes': a list of float32 numpy arrays of shape [M, 4]
'groundtruth_classes': a list of int32 numpy arrays of shape [M]
and the remaining fields below are optional:
'difficult': a list of boolean arrays of shape [M] indicating the
difficulty of groundtruth boxes. Some datasets like PASCAL VOC provide
this information and it is used to remove difficult examples from eval
in order to not penalize the models on them.
Note that it is okay to have additional fields in result_lists --- they
are simply ignored.
categories: a list of dictionaries representing all possible categories.
Each dict in this list has the following keys:
'id': (required) an integer id uniquely identifying this category
'name': (required) string representing category name
e.g., 'cat', 'dog', 'pizza'
label_id_offset: an integer offset for the label space.
iou_thres: float determining the IoU threshold at which a box is considered
correct. Defaults to the standard 0.5.
corloc_summary: boolean. If True, also outputs CorLoc metrics.
nms_type: type of NMS (standard|soft-linear|soft_gaussian)
nms_thres: iou threshold for non maximum suppression.
soft_nms_sigma: Soft NMS sigma.
Returns:
A dictionary of metric names to scalar values.
Raises:
ValueError: if the set of keys in result_lists is not a superset of the
expected list of keys. Unexpected keys are ignored.
ValueError: if the lists in result_lists have inconsistent sizes.
"""
# check for expected keys in result_lists
expected_keys = [
'detection_boxes', 'detection_scores', 'detection_classes', 'image_id'
]
expected_keys += ['groundtruth_boxes', 'groundtruth_classes']
if not set(expected_keys).issubset(set(result_lists.keys())):
raise ValueError('result_lists does not have expected key set.')
num_results = len(result_lists[expected_keys[0]])
for key in expected_keys:
if len(result_lists[key]) != num_results:
raise ValueError('Inconsistent list sizes in result_lists')
if 'groundtruth_subset' in result_lists.keys():
subset_list = result_lists['groundtruth_subset']
subset_names = set()
for subset in subset_list:
for subset2 in subset:
for subset3 in subset2.split('|'):
subset_names.add(subset3)
if not subset_names:
subset_names.add('default')
# Pascal VOC evaluator assumes foreground index starts from zero.
categories = copy.deepcopy(categories)
for idx in range(len(categories)):
categories[idx]['id'] -= label_id_offset
# num_classes (maybe encoded as categories)
num_classes = max([cat['id'] for cat in categories]) + 1
log.infov('Computing Pascal VOC metrics on results.')
if all(image_id.isdigit() for image_id in result_lists['image_id']):
image_ids = [int(image_id) for image_id in result_lists['image_id']]
else:
image_ids = range(num_results)
evaluator = object_detection_evaluation.ObjectDetectionEvaluation(
num_classes,
matching_iou_threshold=iou_thres,
nms_type=nms_type,
nms_iou_threshold=nms_thres,
soft_nms_sigma=soft_nms_sigma,
subset_names=subset_names)
difficult_lists = None
if 'difficult' in result_lists and result_lists['difficult']:
difficult_lists = result_lists['difficult']
for idx, image_id in enumerate(image_ids):
subset = None
if len(result_lists['groundtruth_subset']) > 0 and \
result_lists['groundtruth_subset'][idx].shape[0] \
== result_lists['groundtruth_boxes'][idx].shape[0]:
subset = result_lists['groundtruth_subset'][idx]
if difficult_lists is not None \
and difficult_lists[idx].size:
# build subset using difficult
difficult = difficult_lists[idx].astype(np.bool)
if subset is None:
subset = np.where(difficult, '', 'default')
else:
subset = np.where(difficult, '', subset)
evaluator.add_single_ground_truth_image_info(
image_id, result_lists['groundtruth_boxes'][idx],
result_lists['groundtruth_classes'][idx] - label_id_offset, subset)
evaluator.add_single_detected_image_info(
image_id, result_lists['detection_boxes'][idx],
result_lists['detection_scores'][idx],
result_lists['detection_classes'][idx] - label_id_offset)
if idx % 500 == 0:
log.infov('idx(%d)/total(%d)' % (idx, len(image_ids)))
per_class_ap, mean_ap, per_class_precisions, per_class_recalls, per_class_corloc, mean_corloc = (
evaluator.evaluate())
visualize_pr_curve(per_class_precisions,
per_class_recalls,
global_step,
eval_dir,
nms_type=nms_type,
nms_iou_threshold=nms_thres,
soft_nms_sigma=soft_nms_sigma)
metrics = {
'Subset {:10} mAP@{}IOU'.format(s, iou_thres): mean_ap[s]
for s in mean_ap
}
category_index = label_map_util.create_category_index(categories)
for subset in per_class_ap:
for idx in range(per_class_ap[subset].size):
if idx in category_index:
display_name = ('Subset {:10} mAP@{}IOU/{}'.format(
subset, iou_thres, category_index[idx]['name']))
metrics[display_name] = per_class_ap[subset][idx]
if corloc_summary:
metrics['CorLoc/CorLoc@{}IOU'.format(iou_thres)] = mean_corloc
for idx in range(per_class_corloc.size):
if idx in category_index:
display_name = ('PerformanceByCategory/CorLoc@{}IOU/{}'.format(
iou_thres, category_index[idx]['name']))
metrics[display_name] = per_class_corloc[idx]
return metrics
def repeated_checkpoint_run(tensor_dict,
update_op,
summary_dir,
aggregated_result_processor=None,
batch_processor=None,
checkpoint_dirs=None,
variables_to_restore=None,
restore_fn=None,
num_batches=1,
eval_interval_secs=120,
max_number_of_evaluations=None,
master='',
save_graph=False,
save_graph_dir='',
metric_names_to_values=None,
keys_to_exclude_from_results=(),
eval_config=None,
gpu_fraction=0.0,
categories=[]):
"""Periodically evaluates desired tensors using checkpoint_dirs or restore_fn.
This function repeatedly loads a checkpoint and evaluates a desired
set of tensors (provided by tensor_dict) and hands the resulting numpy
arrays to a function result_processor which can be used to further
process/save/visualize the results.
Args:
tensor_dict: a dictionary holding tensors representing a batch of detections
and corresponding groundtruth annotations.
update_op: a tensorflow update op that will run for each batch along with
the tensors in tensor_dict.
summary_dir: a directory to write metrics summaries.
aggregated_result_processor: a function taking one argument:
1. result_lists: a dictionary with keys matching those in tensor_dict
and corresponding values being the list of results for each tensor
in tensor_dict. The length of each such list is num_batches.
batch_processor: a function taking three arguments:
1. tensor_dict: the same tensor_dict that is passed in as the first
argument to this function.
2. sess: a tensorflow session
3. batch_index: an integer representing the index of the batch amongst
all batches
4. update_op: a tensorflow update op that will run for each batch.
and returns result_dict, a dictionary of results for that batch.
By default, batch_processor is None, which defaults to running:
return sess.run(tensor_dict)
checkpoint_dirs: list of directories to load into a DetectionModel or an
EnsembleModel if restore_fn isn't set. Also used to determine when to run
next evaluation. Must have at least one element.
variables_to_restore: None, or a dictionary mapping variable names found in
a checkpoint to model variables. The dictionary would normally be
generated by creating a tf.train.ExponentialMovingAverage object and
calling its variables_to_restore() method. Not used if restore_fn is set.
restore_fn: a function that takes a tf.Session object and correctly restores
all necessary variables from the correct checkpoint file.
num_batches: the number of batches to use for evaluation.
eval_interval_secs: the number of seconds between each evaluation run.
max_number_of_evaluations: the max number of iterations of the evaluation.
If the value is left as None the evaluation continues indefinitely.
master: the location of the Tensorflow session.
save_graph: whether or not the Tensorflow graph is saved as a pbtxt file.
save_graph_dir: where to save on disk the Tensorflow graph. If store_graph
is True this must be non-empty.
metric_names_to_values: A dictionary containing metric names to tensors
which will be evaluated after processing all batches
of [tensor_dict, update_op]. If any metrics depend on statistics computed
during each batch ensure that `update_op` tensor has a control dependency
on the update ops that compute the statistics.
keys_to_exclude_from_results: keys in tensor_dict that will be excluded
from results_list. Note that the tensors corresponding to these keys will
still be evaluated for each batch, but won't be added to results_list.
gpu_fraction: GPU memory fraction for evaluation.
Raises:
ValueError: if max_num_of_evaluations is not None or a positive number.
ValueError: if checkpoint_dirs doesn't have at least one element.
"""
if max_number_of_evaluations and max_number_of_evaluations <= 0:
raise ValueError(
'`number_of_steps` must be either None or a positive number.')
if not checkpoint_dirs:
raise ValueError('`checkpoint_dirs` must have at least one entry.')
last_evaluated_model_path = None
number_of_evaluations = 0
while True:
start = time.time()
log.infov('Starting evaluation at ' +
time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
model_path = tf.train.latest_checkpoint(checkpoint_dirs[0])
if not model_path:
log.warn('No model found in %s. Will try again in %d seconds',
checkpoint_dirs[0], eval_interval_secs)
elif model_path == last_evaluated_model_path:
log.warn(
'Found already evaluated checkpoint. Will try again in %d '
'seconds', eval_interval_secs)
else:
last_evaluated_model_path = model_path
run_checkpoint_once(tensor_dict, update_op, summary_dir,
aggregated_result_processor, batch_processor,
checkpoint_dirs, variables_to_restore,
restore_fn, num_batches, master, save_graph,
save_graph_dir, metric_names_to_values,
keys_to_exclude_from_results, eval_config,
gpu_fraction, categories)
number_of_evaluations += 1
if (max_number_of_evaluations
and number_of_evaluations >= max_number_of_evaluations):
log.infov('Finished evaluation!')
break
time_to_next_eval = start + eval_interval_secs - time.time()
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
