def evaluate(train_dir,
eval_dir,
config,
dataset_fn,
num_batches,
master=''):
"""Evaluate the model repeatedly."""
tf.gfile.MakeDirs(eval_dir)
_trial_summary(
config.hparams, config.eval_examples_path or config.tfds_name, eval_dir)
with tf.Graph().as_default():
model = config.model
model.build(config.hparams,
config.data_converter.output_depth,
is_training=False)
eval_op = model.eval(
**_get_input_tensors(dataset_fn().take(num_batches), config))
hooks = [
contrib_training.StopAfterNEvalsHook(num_batches),
contrib_training.SummaryAtEndHook(eval_dir)
]
contrib_training.evaluate_repeatedly(
train_dir,
eval_ops=eval_op,
hooks=hooks,
eval_interval_secs=60,
master=master)
def main(_):
FLAGS.properties_file = "properties_test.json"
properties = get_properties(FLAGS)
logdir = setup_logdir(FLAGS, properties)
FLAGS.running_mode = "test"
model = get_model(FLAGS, properties)
with tf.variable_scope('model', reuse=True):
batch = model.data_handler.get_batch(FLAGS.batch_size, FLAGS)
noise = tf.random_uniform([FLAGS.batch_size, FLAGS.z_dim],
minval=-1.0,
maxval=1.0,
dtype=tf.float32,
name='z0')
generated_data = model.get_generated_data(noise, batch[1:])
model.data_handler.display_fake_data(generated_data, batch[1],
FLAGS.batch_size)
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
id_to_enzyme_class_dict = properties["class_mapping"]
evaluate_repeatedly(checkpoint_dir=logdir,
hooks=[
BlastHook(id_to_enzyme_class_dict,
every_n_steps=1,
output_dir=logdir,
n_examples=FLAGS.batch_size,
running_mode=FLAGS.running_mode),
tf.contrib.training.SummaryAtEndHook(logdir),
tf.contrib.training.StopAfterNEvalsHook(1)
],
eval_ops=generated_data,
config=session_config)
def main(_):
FLAGS.properties_file = "properties_test.json"
properties = get_properties(FLAGS)
logdir = setup_logdir(FLAGS, properties)
FLAGS.running_mode = "test"
path = os.path.join(FLAGS.data_dir, FLAGS.dataset.replace("\\", os.sep),
"d_test.csv")
# data_to_test = pd.read_csv(path, header=None)[:FLAGS.batch_size]
# data_to_test = from_amino_acid_to_id(data_to_test, 0)
model = get_model(FLAGS, properties)
path = os.path.join(FLAGS.data_dir, FLAGS.dataset.replace("\\", os.sep),
"properties_test_original.json")
with open(path) as json_data_file:
properties = json.load(json_data_file)
with tf.variable_scope('model', reuse=True):
real_x, labels = model.data_handler.get_batch(FLAGS.batch_size, FLAGS)
real_x, labels = model.data_handler.prepare_real_data(real_x, labels)
#
# data = model.data_handler.get_embedded_seqs(data_to_test)
# d_scores, _ = model.get_discriminator_result(data, batch[1:], reuse=True)
# printing_d_scores = tf.py_func(
# lambda vals, scores: print_protein_seq(vals, properties["class_mapping"], d_scores=scores),
# [tf.squeeze(data_to_test), d_scores], tf.string)
noise = np.random.normal(size=[FLAGS.batch_size, FLAGS.z_dim],
loc=0.0,
scale=1.0)
z = noise
generated_data = model.get_generated_data(
tf.convert_to_tensor(z, dtype=tf.float32), labels)
d_fake_scores, _ = model.get_discriminator_result(generated_data,
labels,
reuse=True)
generated_data_ids = model.data_handler.convert_to_acid_ids(
generated_data)
printing_sequeces = tf.py_func(
lambda vals, labels: print_protein_seq(
vals, properties["class_mapping"], labels=labels),
[tf.squeeze(generated_data_ids), labels[0]], tf.string)
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
evaluate_repeatedly(
checkpoint_dir=logdir,
hooks=[
# BlastHook(id_to_enzyme_class_dict, every_n_steps=1, output_dir=logdir, n_examples=FLAGS.batch_size,
# running_mode=FLAGS.running_mode),
tf.contrib.training.SummaryAtEndHook(logdir),
tf.contrib.training.StopAfterNEvalsHook(1)
],
eval_ops=[printing_sequeces],
max_number_of_evaluations=1,
config=session_config)
def main(_):
FLAGS.properties_file = "properties_test.json"
properties = get_properties(FLAGS)
logdir = setup_logdir(FLAGS, properties)
FLAGS.running_mode = "test"
path = os.path.join(FLAGS.data_dir, FLAGS.dataset.replace("\\", os.sep),
"d_test.csv")
data_to_test = pd.read_csv(path, header=None)[:FLAGS.batch_size]
data_to_test = from_amino_acid_to_id(data_to_test, 0)
model = get_model(FLAGS, properties)
with tf.variable_scope('model', reuse=True):
batch = model.data_handler.get_batch(FLAGS.batch_size, FLAGS)
data = model.data_handler.get_embedded_seqs(data_to_test)
d_scores, _ = model.get_discriminator_result(data,
batch[1:],
reuse=True)
printing_d_scores = tf.py_func(
lambda vals, scores: print_protein_seq(
vals, properties["class_mapping"], d_scores=scores),
[tf.squeeze(data_to_test), d_scores], tf.string)
noise1 = np.random.uniform(size=[FLAGS.z_dim], low=-1.0, high=1.0)
noise2 = np.random.uniform(size=[FLAGS.z_dim], low=-1.0, high=1.0)
z = np.stack([
slerp(ratio, noise1, noise2)
for ratio in np.linspace(0, 1, FLAGS.batch_size)
])
generated_data = model.get_generated_data(
tf.convert_to_tensor(z, dtype=tf.float32), batch[1:])
d_fake_scores, _ = model.get_discriminator_result(generated_data,
batch[1:],
reuse=True)
generated_data_ids = model.data_handler.convert_to_acid_ids(
generated_data)
printing_sequeces = tf.py_func(
lambda vals, scores: print_protein_seq(
vals, properties["class_mapping"], d_scores=scores),
[tf.squeeze(generated_data_ids), d_fake_scores], tf.string)
session_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
evaluate_repeatedly(
checkpoint_dir=logdir,
hooks=[
# BlastHook(id_to_enzyme_class_dict, every_n_steps=1, output_dir=logdir, n_examples=FLAGS.batch_size,
# running_mode=FLAGS.running_mode),
tf.contrib.training.SummaryAtEndHook(logdir),
tf.contrib.training.StopAfterNEvalsHook(1)
],
eval_ops=[printing_d_scores, printing_sequeces],
max_number_of_evaluations=1,
config=session_config)
def run_eval(build_graph_fn,
train_dir,
eval_dir,
num_batches,
timeout_secs=300):
"""Runs the training loop.
Args:
build_graph_fn: A function that builds the graph ops.
train_dir: The path to the directory where checkpoints will be loaded
from for evaluation.
eval_dir: The path to the directory where the evaluation summary events
will be written to.
num_batches: The number of full batches to use for each evaluation step.
timeout_secs: The number of seconds after which to stop waiting for a new
checkpoint.
Raises:
ValueError: If `num_batches` is less than or equal to 0.
"""
if num_batches <= 0:
raise ValueError(
'`num_batches` must be greater than 0. Check that the batch size is '
'no larger than the number of records in the eval set.')
with tf.Graph().as_default():
build_graph_fn()
global_step = tf.train.get_or_create_global_step()
loss = tf.get_collection('loss')[0]
perplexity = tf.get_collection('metrics/perplexity')[0]
accuracy = tf.get_collection('metrics/accuracy')[0]
eval_ops = tf.get_collection('eval_ops')
logging_dict = {
'Global Step': global_step,
'Loss': loss,
'Perplexity': perplexity,
'Accuracy': accuracy
}
hooks = [
EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches),
contrib_training.StopAfterNEvalsHook(num_batches),
contrib_training.SummaryAtEndHook(eval_dir),
]
contrib_training.evaluate_repeatedly(train_dir,
eval_ops=eval_ops,
hooks=hooks,
eval_interval_secs=60,
timeout=timeout_secs)
def Evaluacion(build_graph_fn,
train_dir,
eval_dir,
num_batches,
timeout_secs=300):
tf.compat.v1.logging.set_verbosity('INFO')
if num_batches <= 0:
raise ValueError(
'`num_batches` must be greater than 0. Check that the batch size is '
'no larger than the number of records in the eval set.')
with tf.Graph().as_default():
# Creamos un modelo igual que el del entrenamiento
build_graph_fn()
# Define the summaries to write:
global_step = tf.train.get_or_create_global_step()
loss = tf.get_collection('loss')[0]
perplexity = tf.get_collection('metrics/perplexity')[0]
accuracy = tf.get_collection('metrics/accuracy')[0]
eval_ops = tf.get_collection('eval_ops')
logging_dict = {
'Global Step': global_step,
'Loss': loss,
'Perplexity': perplexity,
'Accuracy': accuracy
}
hooks = [
EvalLoggingTensorHook(logging_dict, every_n_iter=num_batches),
contrib_training.StopAfterNEvalsHook(num_batches),
contrib_training.SummaryAtEndHook(eval_dir),
]
# names_to_values = contrib_training.evaluate_once(
# checkpoint_path=train_dir,
# eval_ops=eval_ops,
# final_ops=logging_dict,
# hooks=hooks,
# config=None)
# for name in names_to_values:
# print('Metric %s has value %f.' % (name, names_to_values[name]))
contrib_training.evaluate_repeatedly(train_dir,
eval_ops=eval_ops,
hooks=hooks,
eval_interval_secs=2,
timeout=timeout_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
model_name=FLAGS.model_name,
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
for eval_scale in FLAGS.eval_scales:
predictions_tag += '_' + str(eval_scale)
if FLAGS.add_flipped_images:
predictions_tag += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False,
with_cls=True,
cls_only=False,
output_valid=True)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
raise NotImplementedError('Multi-scale is not supported yet!')
metric_map = {}
## Extract cls logits
if FLAGS.weakly:
_, end_points = feature_extractor.extract_features(
samples[common.IMAGE],
output_stride=model_options.output_stride,
multi_grid=model_options.multi_grid,
model_variant=model_options.model_variant,
depth_multiplier=model_options.depth_multiplier,
divisible_by=model_options.divisible_by,
reuse=tf.AUTO_REUSE,
is_training=False,
preprocessed_images_dtype=model_options.
preprocessed_images_dtype,
global_pool=True,
num_classes=dataset.num_of_classes - 1)
# ResNet beta version has an additional suffix in FLAGS.model_variant, but
# it shares the same variable names with original version. Add a special
# handling here for beta version ResNet.
logits = end_points['{}/logits'.format(
FLAGS.model_variant).replace('_beta', '')]
logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1])
cls_pred = tf.sigmoid(logits)
# Multi-label classification evaluation
cls_label = samples['cls_label']
cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32)
## For classification
metric_map['eval/cls_overall'] = tf.metrics.accuracy(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_precision'] = tf.metrics.precision(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_recall'] = tf.metrics.recall(
labels=cls_label, predictions=cls_pred)
## For segmentation branch eval
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
# Define the evaluation metric.
num_classes = dataset.num_of_classes
## For segmentation
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset( # 获取验证集图片数据
dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg
split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val”
dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录
batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1
crop_size=[int(sz)
for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513
min_resize_value=FLAGS.min_resize_value, # 默认为None
max_resize_value=FLAGS.max_resize_value, # 默认为None
resize_factor=FLAGS.resize_factor, # 默认为None
model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65
num_readers=2, # 并行读取图片的数量
is_training=False, # 不训练
should_shuffle=False, # 不将输入的数据随机打乱
should_repeat=False) # 不一直重复
tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据
'''
samples:
{'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>,
'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>,
'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>,
'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>,
'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>,
'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>}
'''
model_options = common.ModelOptions( # 模型参数
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
}, # {semantic: 19}
crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048
atrous_rates=FLAGS.atrous_rates, # 6,12,18
output_stride=FLAGS.output_stride) # 16
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape( # 设置形状
[
FLAGS.eval_batch_size, # 默认为1
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]),
3
])
if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options, # 进行每个像素点预测
image_pyramid=FLAGS.image_pyramid)
'''
predictions:
{'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>,
'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>}
'''
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1]) # 预测标签
labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签
weights = tf.to_float(tf.not_equal(labels,
dataset.ignore_label)) # 各标签权重
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou' # MIoU
predictions_tag1 = 'accuracy_pixel' # 像素精度
for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0]
predictions_tag += '_' + str(eval_scale)
predictions_tag1 += '_' + str(eval_scale)
if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型
predictions_tag += '_flipped'
predictions_tag1 += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes # 19
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
'''
metric_map:
{'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
<tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)}
'''
metric_map['eval/%s_overall_accuracy_' %
predictions_tag] = tf.metrics.accuracy(
labels=labels, predictions=predictions, weights=weights)
# IoU for each class.
'''
tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None)
Returns a one-hot tensor.
ndices表示输入的多个数值，通常是矩阵形式；depth表示输出的尺寸。
'''
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes]) # 预测输出的one_hot
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels,
[-1, num_classes]) # 真实label的one_hot
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c
) # miou_1.0_class_c
predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tn, tn_op = tf.metrics.true_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
ap = tf.where(tf.greater(tp + fn, 0.0),
(tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
'''
(metrics_to_values, metrics_to_updates):
({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>,
'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>,
'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>,
'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>,
'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>,
'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>,
'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>,
'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>,
'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>,
'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>,
'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>,
'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>,
'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>,
'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>,
'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>,
'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>,
'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>},
{'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>,
'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>,
'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>,
'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>,
'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>,
'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>,
'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>,
'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>,
'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>,
'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>,
'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>,
'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>,
'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>,
'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>,
'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>,
'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>,
'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>,
'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>,
'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>})
'''
'''
tf.Print(input, data, message=None, first_n=None, summarize=None, name=None)
最低要求两个输入，input和data，input是需要打印的变量的名字，data要求是一个list，里面包含要打印的内容。
'''
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
