def __compute_AP(self, c_scores, c_tp, c_fp, c_num_gbboxes):
aps_voc07 = {}
aps_voc12 = {}
for c in c_scores.keys():
num_gbboxes = c_num_gbboxes[c]
tp = c_tp[c]
fp = c_fp[c]
scores = c_scores[c]
#reshape data
num_gbboxes = math_ops.to_int64(num_gbboxes)
scores = math_ops.to_float(scores)
stype = tf.bool
tp = tf.cast(tp, stype)
fp = tf.cast(fp, stype)
# Reshape TP and FP tensors and clean away 0 class values.(difficult bboxes)
scores = tf.reshape(scores, [-1])
tp = tf.reshape(tp, [-1])
fp = tf.reshape(fp, [-1])
# Remove TP and FP both false.
mask = tf.logical_or(tp, fp)
rm_threshold = 1e-4
mask = tf.logical_and(mask, tf.greater(scores, rm_threshold))
scores = tf.boolean_mask(scores, mask)
tp = tf.boolean_mask(tp, mask)
fp = tf.boolean_mask(fp, mask)
num_gbboxes = tf.reduce_sum(num_gbboxes)
num_detections = tf.size(scores, out_type=tf.int32)
# Precison and recall values.
prec, rec = tfe.precision_recall(num_gbboxes, num_detections, tp,
fp, scores)
v = tfe.average_precision_voc07(prec, rec)
aps_voc07[c] = v
# Average precision VOC12.
v = tfe.average_precision_voc12(prec, rec)
aps_voc12[c] = v
return aps_voc07, aps_voc12
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
# =================================================================== #
# Dataset + SSD model + Pre-processing
# =================================================================== #
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# get the ssd network and its anchors
ssd_cls = ssd.SSDnet
ssd_params = ssd_cls.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_cls(ssd_params)
image_size = ssd_net.params.img_shape
# Evaluation shape and associated anchors: eval_image_size
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.eval_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device('/cpu:0'):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size,
shuffle=False)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes] = provider.get(
['image', 'shape', 'object/label', 'object/bbox'])
if FLAGS.remove_difficult:
[gdifficults] = provider.get(['object/difficult'])
else:
gdifficults = tf.zeros(tf.shape(glabels), dtype=tf.int64)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes, gbbox_img = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT,
resize=FLAGS.eval_resize,
difficults=None)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] * 5 + [len(ssd_anchors)] * 3
# Evaluation batch.
r = tf.train.batch(tf_utils.reshape_list([
image, glabels, gbboxes, gdifficults, gbbox_img, gclasses,
glocalisations, gscores
]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size,
dynamic_pad=True)
(b_image, b_glabels, b_gbboxes, b_gdifficults, b_gbbox_img,
b_gclasses, b_glocalisations,
b_gscores) = tf_utils.reshape_list(r, batch_shape)
# =================================================================== #
# SSD Network + Ouputs decoding.
# =================================================================== #
dict_metrics = {}
arg_scope = ssd_net.arg_scope(data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=False)
# Add losses functions.
# pdb.set_trace()
ssd_net.losses(logits, localisations, b_gclasses, b_glocalisations,
b_gscores)
# Performing post-processing on CPU: loop-intensive, usually more efficient.
with tf.device('/device:CPU:0'):
# Detected objects from SSD output.
localisations = ssd_net.bboxes_decode(localisations, ssd_anchors)
rscores, rbboxes = \
ssd_net.detected_bboxes(predictions, localisations,
select_threshold=FLAGS.select_threshold,
nms_threshold=FLAGS.nms_threshold,
clipping_bbox=None,
top_k=FLAGS.select_top_k,
keep_top_k=FLAGS.keep_top_k)
# Compute TP and FP statistics.
num_gbboxes, tp, fp, rscores = \
tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
b_glabels, b_gbboxes, b_gdifficults,
matching_threshold=FLAGS.matching_threshold)
# Variables to restore: moving avg. or normal weights.
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
# =================================================================== #
# Evaluation metrics.
# =================================================================== #
with tf.device('/device:CPU:0'):
dict_metrics = {}
# First add all losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Extra losses as well.
for loss in tf.get_collection('EXTRA_LOSSES'):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Add metrics to summaries and Print on screen.
for name, metric in dict_metrics.items():
# summary_name = 'eval/%s' % name
summary_name = name
op = tf.summary.scalar(summary_name, metric[0], collections=[])
# op = tf.Print(op, [metric[0]], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# FP and TP metrics.
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp,
rscores)
for c in tp_fp_metric[0].keys():
dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c],
tp_fp_metric[1][c])
# Add to summaries precision/recall values.
aps_voc07 = {}
aps_voc12 = {}
for c in tp_fp_metric[0].keys():
# Precison and recall values.
prec, rec = tfe.precision_recall(*tp_fp_metric[0][c])
# Average precision VOC07.
v = tfe.average_precision_voc07(prec, rec)
summary_name = 'AP_VOC07/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc07[c] = v
# Average precision VOC12.
v = tfe.average_precision_voc12(prec, rec)
summary_name = 'AP_VOC12/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc12[c] = v
# Mean average precision VOC07.
summary_name = 'AP_VOC07/mAP'
mAP = tf.add_n(list(aps_voc07.values())) / len(aps_voc07)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Mean average precision VOC12.
summary_name = 'AP_VOC12/mAP'
mAP = tf.add_n(list(aps_voc12.values())) / len(aps_voc12)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# for i, v in enumerate(l_precisions):
# summary_name = 'eval/precision_at_recall_%.2f' % LIST_RECALLS[i]
# op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
# tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Split into values and updates ops.
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(
dict_metrics)
# =================================================================== #
# Evaluation loop.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
# config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
# Number of batches...
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
num_batches = math.ceil(dataset.num_samples /
float(FLAGS.batch_size))
if not FLAGS.wait_for_checkpoints:
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(
FLAGS.checkpoint_path)
else:
checkpoint_path = os.path.join(os.getcwd(),
FLAGS.checkpoint_path)
tf.logging.info('Evaluating %s' % checkpoint_path)
# Standard evaluation loop.
start = time.time()
# pdb.set_trace()
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=flatten(list(names_to_updates.values())),
variables_to_restore=variables_to_restore,
session_config=config)
# Log time spent.
elapsed = time.time()
elapsed = elapsed - start
print('Time spent : %.3f seconds.' % elapsed)
print('Time spent per BATCH: %.3f seconds.' %
(elapsed / num_batches))
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Waiting loop.
slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=flatten(list(names_to_updates.values())),
variables_to_restore=variables_to_restore,
eval_interval_secs=60,
max_number_of_evaluations=np.inf,
session_config=config,
timeout=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
# Get the RON network and its anchors.
ron_class = nets_factory.get_network(FLAGS.model_name)
ron_params = ron_class.default_params._replace(num_classes=FLAGS.num_classes)
ron_net = ron_class(ron_params)
ron_shape = ron_net.params.img_shape
ron_anchors = ron_net.anchors(ron_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
tf_utils.print_configuration(FLAGS.__flags, ron_params,
dataset.data_sources, FLAGS.eval_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device('/cpu:0'):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size,
shuffle=False)
# Get for SSD network: image, labels, bboxes.
[image_, shape, glabels, gbboxes, gdifficults] = provider.get(['image', 'shape',
'object/label',
'object/bbox',
'object/difficult'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes, gbbox_img = \
image_preprocessing_fn(image_, glabels, gbboxes,
out_shape=ron_shape,
data_format=DATA_FORMAT,
difficults=None)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores, _ = \
ron_net.bboxes_encode(glabels, gbboxes, ron_anchors)
batch_shape = [1] * 5 + [len(ron_anchors)] * 3
# Evaluation batch.
r = tf.train.batch(
tf_utils.reshape_list([image, glabels, gbboxes, gdifficults, gbbox_img,
gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size,
dynamic_pad=True)
(b_image, b_glabels, b_gbboxes, b_gdifficults, b_gbbox_img, b_gclasses,
b_glocalisations, b_gscores) = tf_utils.reshape_list(r, batch_shape)
# =================================================================== #
# SSD Network + Ouputs decoding.
# =================================================================== #
dict_metrics = {}
arg_scope = ron_net.arg_scope(weight_decay=FLAGS.weight_decay,
is_training=False,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, logits, objness_pred, objness_logits, localisations, end_points = \
ron_net.net(b_image, is_training=False)
# Add loss function.
ron_net.losses(logits, localisations, objness_logits, objness_pred,
b_gclasses, b_glocalisations, b_gscores,
match_threshold = FLAGS.match_threshold,
neg_threshold = FLAGS.neg_threshold,
objness_threshold = FLAGS.objectness_thres,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
beta=FLAGS.loss_beta,
label_smoothing=FLAGS.label_smoothing)
variables_to_restore = slim.get_variables_to_restore()
# Performing post-processing on CPU: loop-intensive, usually more efficient.
with tf.device('/device:CPU:0'):
# Detected objects from SSD output.
localisations = ron_net.bboxes_decode(localisations, ron_anchors)
filtered_predictions = []
for i, objness in enumerate(objness_pred):
filtered_predictions.append(tf.cast(tf.greater(objness, FLAGS.objectness_thres), tf.float32) * predictions[i])
rscores, rbboxes = \
ron_net.detected_bboxes(filtered_predictions, localisations,
select_threshold=FLAGS.select_threshold,
nms_threshold=FLAGS.nms_threshold,
clipping_bbox=[0., 0., 1., 1.],
top_k=FLAGS.select_top_k,
keep_top_k=FLAGS.keep_top_k)
labels_list = []
for k, v in rscores.items():
labels_list.append(tf.ones_like(v, tf.int32) * k)
save_image_op = tf.py_func(save_image_with_bbox,
[tf.cast(tf.squeeze(b_image, 0), tf.float32),
tf.squeeze(tf.concat(labels_list, axis=1), 0),
#tf.convert_to_tensor(list(rscores.keys()), dtype=tf.int64),
tf.squeeze(tf.concat(list(rscores.values()), axis=1), 0),
tf.squeeze(tf.concat(list(rbboxes.values()), axis=1), 0)],
tf.int64, stateful=True)
with tf.control_dependencies([save_image_op]):
# Compute TP and FP statistics.
num_gbboxes, tp, fp, rscores = \
tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
b_glabels, b_gbboxes, b_gdifficults,
matching_threshold=0.5)
# =================================================================== #
# Evaluation metrics.
# =================================================================== #
with tf.device('/device:CPU:0'):
dict_metrics = {}
# First add all losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Extra losses as well.
for loss in tf.get_collection('EXTRA_LOSSES'):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Add metrics to summaries and Print on screen.
for name, metric in dict_metrics.items():
# summary_name = 'eval/%s' % name
summary_name = name
op = tf.summary.scalar(summary_name, metric[0], collections=[])
# op = tf.Print(op, [metric[0]], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# FP and TP metrics.
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp, rscores)
metrics_name = ('nobjects', 'ndetections', 'tp', 'fp', 'scores')
for c in tp_fp_metric[0].keys():
for _ in range(len(tp_fp_metric[0][c])):
dict_metrics['tp_fp_%s_%s' % (c, metrics_name[_])] = (tp_fp_metric[0][c][_],
tp_fp_metric[1][c][_])
# for c in tp_fp_metric[0].keys():
# dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c],
# tp_fp_metric[1][c])
# Add to summaries precision/recall values.
aps_voc07 = {}
aps_voc12 = {}
for c in tp_fp_metric[0].keys():
# Precison and recall values.
prec, rec = tfe.precision_recall(*tp_fp_metric[0][c])
# Average precision VOC07.
v = tfe.average_precision_voc07(prec, rec)
summary_name = 'AP_VOC07/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc07[c] = v
# Average precision VOC12.
v = tfe.average_precision_voc12(prec, rec)
summary_name = 'AP_VOC12/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc12[c] = v
# Mean average precision VOC07.
summary_name = 'AP_VOC07/mAP'
mAP = tf.add_n(list(aps_voc07.values())) / len(aps_voc07)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Mean average precision VOC12.
summary_name = 'AP_VOC12/mAP'
mAP = tf.add_n(list(aps_voc12.values())) / len(aps_voc12)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# for i, v in enumerate(l_precisions):
# summary_name = 'eval/precision_at_recall_%.2f' % LIST_RECALLS[i]
# op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
# tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Split into values and updates ops.
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(dict_metrics)
# =================================================================== #
# Evaluation loop.
# =================================================================== #
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options)
# config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
# Number of batches...
num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size))
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Standard evaluation loop.
start = time.time()
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
session_config=config)
# Log time spent.
elapsed = time.time()
elapsed = elapsed - start
print('Time spent : %.3f seconds.' % elapsed)
print('Time spent per BATCH: %.3f seconds.' % (elapsed / num_batches))
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
# initalize the net
network_fn = nets_factory.get_network(FLAGS.model_name)
net = network_fn()
out_shape = net.params.img_shape
out_shape = (300, 300)
anchors = net.anchors(out_shape)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device('/cpu:0'):
b_image, glabels, b_gbboxes, g_bbox_img, b_glocalisations, b_gscores =\
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS,
file_pattern = '*.tfrecord',
is_training = False,
shuffe = FLAGS.shuffle_data)
b_gdifficults = tf.zeros(tf.shape(glabels), dtype=tf.int64)
dict_metrics = {}
arg_scope = net.arg_scope(data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=False, use_batch=FLAGS.use_batch)
# Add losses functions.
#total_loss = net.losses(logits, localisations,
# b_glocalisations, b_gscores)
predictions = []
for i in range(len(logits)):
predictions.append(slim.softmax(logits[i]))
# Performing post-processing on CPU: loop-intensive, usually more efficient.
with tf.device('/device:CPU:0'):
# Detected objects from SSD output.
localisations = net.bboxes_decode(localisations, anchors)
rscores, rbboxes = \
net.detected_bboxes(predictions, localisations,
select_threshold=FLAGS.select_threshold,
nms_threshold=FLAGS.nms_threshold,
clipping_bbox=None,
top_k=FLAGS.select_top_k,
keep_top_k=FLAGS.keep_top_k)
# Compute TP and FP statistics.
num_gbboxes, tp, fp, rscores = \
tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
glabels, b_gbboxes, b_gdifficults,
matching_threshold=FLAGS.matching_threshold)
# Variables to restore: moving avg. or normal weights.
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
# =================================================================== #
# Evaluation metrics.
# =================================================================== #
with tf.device(FLAGS.gpu_eval):
dict_metrics = {}
# Extra losses as well.
for loss in tf.get_collection('EXTRA_LOSSES'):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Add metrics to summaries and Print on screen.
for name, metric in dict_metrics.items():
# summary_name = 'eval/%s' % name
summary_name = name
op = tf.summary.scalar(summary_name, metric[0], collections=[])
# op = tf.Print(op, [metric[0]], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# FP and TP metrics.
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp,
rscores)
for c in tp_fp_metric[0].keys():
dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c],
tp_fp_metric[1][c])
# Add to summaries precision/recall values.
icdar2013 = {}
for c in tp_fp_metric[0].keys():
# Precison and recall values.
prec, rec = tfe.precision_recall(*tp_fp_metric[0][c])
op = tf.summary.scalar('precision',
tf.reduce_mean(prec),
collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
op = tf.summary.scalar('recall',
tf.reduce_mean(rec),
collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Average precision VOC07.
v = tfe.average_precision_voc12(prec, rec)
#v = (prec + rec)/2.
summary_name = 'ICDAR13/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
icdar2013[c] = v
# Mean average precision VOC07.
summary_name = 'ICDAR13/mAP'
mAP = tf.add_n(list(icdar2013.values())) / len(icdar2013)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Split into values and updates ops.
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(
dict_metrics)
# =================================================================== #
# Evaluation loop.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
# config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
# Number of batches...
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
num_batches = math.ceil(FLAGS.num_samples /
float(FLAGS.batch_size))
if not FLAGS.wait_for_checkpoints:
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(
FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Standard evaluation loop.
start = time.time()
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
session_config=config)
# Log time spent.
elapsed = time.time()
elapsed = elapsed - start
print('Time spent : %.3f seconds.' % elapsed)
print('Time spent per BATCH: %.3f seconds.' %
(elapsed / num_batches))
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Waiting loop.
slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
eval_interval_secs=60,
max_number_of_evaluations=np.inf,
session_config=config,
timeout=None)
def get_mAP_tf_accumulative(self, predictions, localisations, glabels,
gbboxes, gdifficults):
# Performing post-processing on CPU: loop-intensive, usually more efficient.
with tf.device('/device:CPU:0'):
# Detected objects from SSD output.
localisations = g_ssd_model.decode_bboxes_all_ayers_tf(
localisations)
rscores, rbboxes = g_ssd_model.detected_bboxes(
predictions, localisations)
# Compute TP and FP statistics.
num_gbboxes, tp, fp, rscores = \
tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
glabels, gbboxes, gdifficults)
dict_metrics = {}
with tf.device('/device:CPU:0'):
# FP and TP metrics.
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp,
rscores)
for c in tp_fp_metric[0].keys():
dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c],
tp_fp_metric[1][c])
# Add to summaries precision/recall values.
aps_voc07 = {}
aps_voc12 = {}
for c in tp_fp_metric[0].keys():
# Precison and recall values.
prec, rec = tfe.precision_recall(*tp_fp_metric[0][c])
# Average precision VOC07.
v = tfe.average_precision_voc07(prec, rec)
summary_name = 'AP_VOC07/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc07[c] = v
# Average precision VOC12.
v = tfe.average_precision_voc12(prec, rec)
summary_name = 'AP_VOC12/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc12[c] = v
# Mean average precision VOC07.
summary_name = 'AP_VOC07/mAP_accumulative'
mAP = tf.add_n(list(aps_voc07.values())) / len(aps_voc07)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Mean average precision VOC12.
summary_name = 'AP_VOC12/mAP_accumulative'
mAP = tf.add_n(list(aps_voc12.values())) / len(aps_voc12)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Split into values and updates ops.
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(
dict_metrics)
return names_to_updates
def ssd_eval(dataset_name, dataset_dir, batch_size, eval_dir):
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
# Dataset + SSD Model + Pre-processing
dataset = dataset_factory.get_dataset(dataset_name, 'test', dataset_dir)
ssd_net = ssd_vgg_300.SSDNet()
ssd_shape = net_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Create a dataset provider and batches
with tf.device('/cpu:0'):
with tf.name_scope(dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
common_queue_capacity = 2 * batch_size,
common_queue_min = batch_size,
shuffle = False
)
[image, shape, glabels, gbboxes] = provider.get(['image', 'shape','object/label', 'object/bbox'])
[gdifficults] = provider.get(['object/difficult'])
image, glabels, gbboxes, gbbox_img = ssd_vgg_preprocessing.preprocess_for_eval(image, glabels, gbboxes, ssd_shape,
data_format = data_format,
resize = ssd_vgg_preprocessing.Resize.WARP_RESIZE)
gclasses, glocalizations, gscores = ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] * 5 + [len(ssd_anchors)] * 3
# Evaluation Batch
r = tf.train.batch(reshape_list([image, glabels, gbboxes, gdifficults, gbbox_img, gclasses, glocalizations, gscores]),
batch_size = batch_size,
num_threads = 1,
capacity = 5 * batch_size,
dynamic_pad = True)
(b_image, b_glabels, b_gbboxes, b_gdifficults, b_gbbox_img, b_gclasses, b_glocalizations,
b_gscores) = reshape_list(r, batch_shape)
# SSD network + output decoding
arg_scope = ssd_net.arg_scope(data_format= data_format)
with slim.arg_scope(arg_scope):
predictions, localizations, logits, _ = ssd_net.net(b_image, is_training=False)
ssd_net.losses(logits, localizations, b_gclasses, b_glocalizations, b_gscores)
with tf.device('/device:CPU:0'):
localizations = ssd_net.bboxes_decode(localizations, ssd_anchors)
rscores, rbboxes = ssd_net.detected_bboxes(predictions, localizations,
select_threshold=0.01,
nms_threshold=0.45,
clipping_bbox=None,
top_k=400,
keep_top_k=200)
num_gbboxes, tp, fp, rscores = tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
b_glabels, b_gbboxes, b_gdifficults,
matching_threshold= 0.5)
variables_to_restore = slim.get_variables_to_restore()
with tf.device('/device:CPU:0'):
dict_metrics = {}
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
for loss in tf.get_collection('EXTRA_LOSSES'):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
for name, metric in dict_metrics.items():
summary_name = name
op = tf.summary.scalar(summary_name, metric[0], collections=[])
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp, rscores)
for c in tp_fp_metric[0].keys():
dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c], tp_fp_metric[1][c])
aps_VOC07 = {}
aps_voc12 = {}
for c in tp_fp_metric[0].keys():
# precision and recall values
pre, rec = tfe.precision_recall(*tp_fp_metric[0][c])
# average precision VOC07
v = tfe.average_precision_voc07(pre,rec)
summary_name = 'AP_VOC07/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_VOC07[c] = v
# Average precision VOC12.
v = tfe.average_precision_voc12(pre, rec)
summary_name = 'AP_VOC12/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc12[c] = v
# Mean average Precision VOC07
summary_name = 'AP_VOC07/mAP'
mAP = tf.add_n(list(aps_VOC07.values()))/len(aps_VOC07)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Mean average precision VOC12.
summary_name = 'AP_VOC12/mAP'
mAP = tf.add_n(list(aps_voc12.values())) / len(aps_voc12)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(dict_metrics)
# Evaluation Loop
gpu_options = tf.GPUOptions(allow_growth=True, per_process_gpu_memory_fraction=0.9)
config = tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options)
num_batches = math.ceil(dataset.num_samples / float(batch_size))
tf.logging.info('Evaluating %s' % ckpt_filename)
start = time.time()
slim.evaluation.evaluate_once(master= '',
checkpoint_path = ckpt_filename,
logdir= eval_dir,
num_evals= num_batches,
eval_op= flatten(list(names_to_updates.values())),
variables_to_restore= variables_to_restore,
session_config = config)
# log time spent
elapsed = time.time() - start
print('Time Spent: %.3f' % elapsed)
print('Time Spent per batch: %.3f seconds' % (elapsed/num_batches))