def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Generate the model
###########################################################
outputs = create_generator(config, input_utils.get_data_shape(config.dataset))
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
# Generate the canvases that lead to the final output image
summaries = []
summaries.extend(layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS))
with tf.name_scope('canvases'):
for step, canvas in enumerate(outputs):
canvas = input_utils.reshape_images(canvas, config.dataset)
tiled_images = image_utils.tile_images(canvas)
summaries.append(tf.summary.image('step{0}'.format(step), tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_ops = tf.group(*outputs)
hooks = [
training.SummaryAtEndHook(FLAGS.log_dir, summary_op),
training.StopAfterNEvalsHook(FLAGS.count)]
training.evaluate_once(checkpoint_path, hooks=hooks, eval_ops=eval_ops)
def evaluate(train_dir,
eval_dir,
config,
dataset_fn,
num_batches,
master=''):
"""Evaluate the model repeatedly."""
tf.gfile.MakeDirs(eval_dir)
_trial_summary(
config.hparams, config.eval_examples_path or config.tfds_name, eval_dir)
with tf.Graph().as_default():
model = config.model
model.build(config.hparams,
config.data_converter.output_depth,
is_training=False)
eval_op = model.eval(
**_get_input_tensors(dataset_fn().take(num_batches), config))
hooks = [
contrib_training.StopAfterNEvalsHook(num_batches),
contrib_training.SummaryAtEndHook(eval_dir)
]
contrib_training.evaluate_repeatedly(
train_dir,
eval_ops=eval_op,
hooks=hooks,
eval_interval_secs=60,
master=master)
def run_eval(build_graph_fn,
train_dir,
eval_dir,
num_batches,
timeout_secs=300):
"""Runs the training loop.
Args:
build_graph_fn: A function that builds the graph ops.
train_dir: The path to the directory where checkpoints will be loaded
from for evaluation.
eval_dir: The path to the directory where the evaluation summary events
will be written to.
num_batches: The number of full batches to use for each evaluation step.
timeout_secs: The number of seconds after which to stop waiting for a new
checkpoint.
Raises:
ValueError: If `num_batches` is less than or equal to 0.
"""
if num_batches <= 0:
raise ValueError(
'`num_batches` must be greater than 0. Check that the batch size is '
'no larger than the number of records in the eval set.')
with tf.Graph().as_default():
build_graph_fn()
global_step = tf.train.get_or_create_global_step()
loss = tf.get_collection('loss')[0]
perplexity = tf.get_collection('metrics/perplexity')[0]
accuracy = tf.get_collection('metrics/accuracy')[0]
eval_ops = tf.get_collection('eval_ops')
logging_dict = {
'Global Step': global_step,
'Loss': loss,
'Perplexity': perplexity,
'Accuracy': accuracy
}
hooks = [
EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches),
contrib_training.StopAfterNEvalsHook(num_batches),
contrib_training.SummaryAtEndHook(eval_dir),
]
contrib_training.evaluate_repeatedly(train_dir,
eval_ops=eval_ops,
hooks=hooks,
eval_interval_secs=60,
timeout=timeout_secs)
def Evaluacion(build_graph_fn,
train_dir,
eval_dir,
num_batches,
timeout_secs=300):
tf.compat.v1.logging.set_verbosity('INFO')
if num_batches <= 0:
raise ValueError(
'`num_batches` must be greater than 0. Check that the batch size is '
'no larger than the number of records in the eval set.')
with tf.Graph().as_default():
# Creamos un modelo igual que el del entrenamiento
build_graph_fn()
# Define the summaries to write:
global_step = tf.train.get_or_create_global_step()
loss = tf.get_collection('loss')[0]
perplexity = tf.get_collection('metrics/perplexity')[0]
accuracy = tf.get_collection('metrics/accuracy')[0]
eval_ops = tf.get_collection('eval_ops')
logging_dict = {
'Global Step': global_step,
'Loss': loss,
'Perplexity': perplexity,
'Accuracy': accuracy
}
hooks = [
EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches),
contrib_training.StopAfterNEvalsHook(num_batches),
contrib_training.SummaryAtEndHook(eval_dir),
]
# names_to_values = contrib_training.evaluate_once(
# checkpoint_path=train_dir,
# eval_ops=eval_ops,
# final_ops=logging_dict,
# hooks=hooks,
# config=None)
# for name in names_to_values:
# print('Metric %s has value %f.' % (name, names_to_values[name]))
contrib_training.evaluate_repeatedly(train_dir,
eval_ops=eval_ops,
hooks=hooks,
eval_interval_secs=2,
timeout=timeout_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
model_name=FLAGS.model_name,
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
for eval_scale in FLAGS.eval_scales:
predictions_tag += '_' + str(eval_scale)
if FLAGS.add_flipped_images:
predictions_tag += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False,
with_cls=True,
cls_only=False,
output_valid=True)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
raise NotImplementedError('Multi-scale is not supported yet!')
metric_map = {}
## Extract cls logits
if FLAGS.weakly:
_, end_points = feature_extractor.extract_features(
samples[common.IMAGE],
output_stride=model_options.output_stride,
multi_grid=model_options.multi_grid,
model_variant=model_options.model_variant,
depth_multiplier=model_options.depth_multiplier,
divisible_by=model_options.divisible_by,
reuse=tf.AUTO_REUSE,
is_training=False,
preprocessed_images_dtype=model_options.
preprocessed_images_dtype,
global_pool=True,
num_classes=dataset.num_of_classes - 1)
# ResNet beta version has an additional suffix in FLAGS.model_variant, but
# it shares the same variable names with original version. Add a special
# handling here for beta version ResNet.
logits = end_points['{}/logits'.format(
FLAGS.model_variant).replace('_beta', '')]
logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1])
cls_pred = tf.sigmoid(logits)
# Multi-label classification evaluation
cls_label = samples['cls_label']
cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32)
## For classification
metric_map['eval/cls_overall'] = tf.metrics.accuracy(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_precision'] = tf.metrics.precision(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_recall'] = tf.metrics.recall(
labels=cls_label, predictions=cls_pred)
## For segmentation branch eval
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
# Define the evaluation metric.
num_classes = dataset.num_of_classes
## For segmentation
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir,
config.dataset,
config.datasubset,
num_folds=config.fold_count,
fold=config.fold,
holdout=True)
init_op, init_feed_dict, image = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images = tf.train.batch([image],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for loss in tf.losses.get_losses():
metrics_map[loss.op.name] = metrics.streaming_mean(loss)
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
total_loss = tf.losses.get_total_loss()
metrics_map[total_loss.op.name] = metrics.streaming_mean(total_loss)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(tf.GraphKeys.MODEL_VARIABLES))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.RNN_OUTPUTS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
images = input_utils.reshape_images(images, config.dataset)
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
# Generate the canvases that lead to the final output image
with tf.name_scope('canvases'):
for step, canvas in enumerate(outputs):
canvas = input_utils.reshape_images(canvas, config.dataset)
tiled_images = image_utils.tile_images(canvas)
summaries.append(
tf.summary.image('step{0}'.format(step), tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
hooks = [
training.SummaryAtEndHook(log_dir=FLAGS.log_dir,
summary_op=summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
eval_fn(checkpoint_path, hooks=hooks, eval_ops=eval_ops, **eval_kwargs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset( # 获取验证集图片数据
dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg
split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val”
dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录
batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1
crop_size=[int(sz)
for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513
min_resize_value=FLAGS.min_resize_value, # 默认为None
max_resize_value=FLAGS.max_resize_value, # 默认为None
resize_factor=FLAGS.resize_factor, # 默认为None
model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65
num_readers=2, # 并行读取图片的数量
is_training=False, # 不训练
should_shuffle=False, # 不将输入的数据随机打乱
should_repeat=False) # 不一直重复
tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据
'''
samples:
{'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>,
'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>,
'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>,
'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>,
'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>,
'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>}
'''
model_options = common.ModelOptions( # 模型参数
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
}, # {semantic: 19}
crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048
atrous_rates=FLAGS.atrous_rates, # 6,12,18
output_stride=FLAGS.output_stride) # 16
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape( # 设置形状
[
FLAGS.eval_batch_size, # 默认为1
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]),
3
])
if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options, # 进行每个像素点预测
image_pyramid=FLAGS.image_pyramid)
'''
predictions:
{'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>,
'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>}
'''
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1]) # 预测标签
labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签
weights = tf.to_float(tf.not_equal(labels,
dataset.ignore_label)) # 各标签权重
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou' # MIoU
predictions_tag1 = 'accuracy_pixel' # 像素精度
for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0]
predictions_tag += '_' + str(eval_scale)
predictions_tag1 += '_' + str(eval_scale)
if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型
predictions_tag += '_flipped'
predictions_tag1 += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes # 19
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
'''
metric_map:
{'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
<tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)}
'''
metric_map['eval/%s_overall_accuracy_' %
predictions_tag] = tf.metrics.accuracy(
labels=labels, predictions=predictions, weights=weights)
# IoU for each class.
'''
tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None)
Returns a one-hot tensor.
ndices表示输入的多个数值，通常是矩阵形式；depth表示输出的尺寸。
'''
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes]) # 预测输出的one_hot
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels,
[-1, num_classes]) # 真实label的one_hot
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c
) # miou_1.0_class_c
predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tn, tn_op = tf.metrics.true_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
ap = tf.where(tf.greater(tp + fn, 0.0),
(tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
'''
(metrics_to_values, metrics_to_updates):
({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>,
'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>,
'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>,
'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>,
'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>,
'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>,
'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>,
'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>,
'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>,
'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>,
'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>,
'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>,
'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>,
'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>,
'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>,
'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>,
'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>},
{'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>,
'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>,
'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>,
'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>,
'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>,
'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>,
'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>,
'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>,
'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>,
'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>,
'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>,
'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>,
'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>,
'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>,
'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>,
'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>,
'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>,
'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>,
'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>})
'''
'''
tf.Print(input, data, message=None, first_n=None, summarize=None, name=None)
最低要求两个输入，input和data，input是需要打印的变量的名字，data要求是一个list，里面包含要打印的内容。
'''
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def evaluate_model(config):
""" Train the model using the passed in config """
###########################################################
# Create the input pipeline
###########################################################
with tf.name_scope('input_pipeline'):
dataset = input_utils.get_dataset(config.datadir, config.dataset,
config.datasubset)
init_op, init_feed_dict, image, label = input_utils.get_data(
config.dataset,
dataset,
config.batch_size,
num_epochs=config.num_epochs,
num_readers=config.num_readers)
images, labels = tf.train.batch(
[image, label],
config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
###########################################################
# Generate the model
###########################################################
outputs = create_model(config, images, dataset)
tfprof.model_analyzer.print_model_analysis(tf.get_default_graph())
###########################################################
# Setup the evaluation metrics and summaries
###########################################################
summaries = []
metrics_map = {}
for metric in tf.get_collection(graph_utils.GraphKeys.METRICS):
metrics_map[metric.op.name] = metrics.streaming_mean(metric)
predictions = tf.argmax(outputs, 1)
metrics_map['accuracy'] = metrics.streaming_accuracy(predictions, labels)
metrics_map['recall_5'] = metrics.streaming_sparse_recall_at_k(
outputs, tf.expand_dims(labels, 1), 5)
names_to_values, names_to_updates = metrics.aggregate_metric_map(
metrics_map)
# Create summaries of the metrics and print them to the screen
for name, value in names_to_values.iteritems():
summary = tf.summary.scalar(name, value, collections=[])
summaries.append(tf.Print(summary, [value], name))
summaries.extend(layers.summarize_collection(
graph_utils.GraphKeys.METRICS))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.QUANTIZED_VARIABLES))
summaries.extend(
layers.summarize_collection(graph_utils.GraphKeys.TRAINING_PARAMETERS))
tiled_images = image_utils.tile_images(images)
summaries.append(tf.summary.image('input_batch', tiled_images))
summary_op = tf.summary.merge(summaries, name='summaries')
###########################################################
# Begin evaluation
###########################################################
checkpoint_path = FLAGS.checkpoint_path
eval_ops = tf.group(*names_to_updates.values())
scaffold = tf.train.Scaffold(init_op, init_feed_dict)
hooks = [
training.SummaryAtEndHook(FLAGS.log_dir, summary_op),
training.StopAfterNEvalsHook(
math.ceil(dataset.num_samples / float(config.batch_size)))
]
eval_kwargs = {}
eval_fn = training.evaluate_repeatedly
if FLAGS.once:
if tf.gfile.IsDirectory(checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(checkpoint_path)
eval_fn = training.evaluate_once
else:
assert tf.gfile.IsDirectory(checkpoint_path), (
'checkpoint path must be a directory when using loop evaluation')
# On Tensorflow master fd87896 fixes this, but for now just set a very large number
eval_kwargs['max_number_of_evaluations'] = sys.maxint
eval_fn(checkpoint_path,
scaffold=scaffold,
hooks=hooks,
eval_ops=eval_ops,
**eval_kwargs)
