def build_model():
"""Build the mobilenet_v1 model for evaluation.
Returns:
g: graph with rewrites after insertion of quantization ops and batch norm
folding.
eval_ops: eval ops for inference.
variables_to_restore: List of variables to restore from checkpoint.
"""
g = tf.Graph()
with g.as_default():
inputs, labels = imagenet_input(is_training=False)
scope = mobilenet_v1.mobilenet_v1_arg_scope(is_training=False,
weight_decay=0.0)
with slim.arg_scope(scope):
logits, _ = mobilenet_v1.mobilenet_v1(
inputs,
is_training=False,
depth_multiplier=FLAGS.depth_multiplier,
num_classes=FLAGS.num_classes,
final_endpoint=FLAGS.final_endpoint)
if FLAGS.quantize:
contrib_quantize.create_eval_graph()
eval_ops = metrics(logits, labels)
return g, eval_ops
def add_final_retrain_ops(class_count, final_tensor_name, bottleneck_tensor,
quantize_layer, is_training):
batch_size, bottleneck_tensor_size = bottleneck_tensor.get_shape().as_list()
assert batch_size is None, 'We want to work with arbitrary batch size.'
with tf.name_scope('input'):
bottleneck_input = tf.placeholder_with_default(
bottleneck_tensor,
shape=[batch_size, bottleneck_tensor_size],
name='BottleneckInputPlaceholder')
ground_truth_input = tf.placeholder(
tf.int64, [batch_size], name='GroundTruthInput')
# Organizing the following ops so they are easier to see in TensorBoard.
layer_name = 'final_retrain_ops'
with tf.name_scope(layer_name):
with tf.name_scope('weights'):
initial_value = tf.truncated_normal(
[bottleneck_tensor_size, class_count], stddev=0.001)
layer_weights = tf.Variable(initial_value, name='final_weights')
variable_summaries(layer_weights)
with tf.name_scope('biases'):
layer_biases = tf.Variable(tf.zeros([class_count]), name='final_biases')
variable_summaries(layer_biases)
with tf.name_scope('Wx_plus_b'):
logits = tf.matmul(bottleneck_input, layer_weights) + layer_biases
tf.summary.histogram('pre_activations', logits)
final_tensor = tf.nn.softmax(logits, name=final_tensor_name)
if quantize_layer:
if is_training:
contrib_quantize.create_training_graph()
else:
contrib_quantize.create_eval_graph()
tf.summary.histogram('activations', final_tensor)
if not is_training:
return None, None, bottleneck_input, ground_truth_input, final_tensor
with tf.name_scope('cross_entropy'):
cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
labels=ground_truth_input, logits=logits)
tf.summary.scalar('cross_entropy', cross_entropy_mean)
with tf.name_scope('train'):
optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
train_step = optimizer.minimize(cross_entropy_mean)
return (train_step, cross_entropy_mean, bottleneck_input, ground_truth_input,
final_tensor)
def main(_):
if not FLAGS.output_file:
raise ValueError(
'You must supply the path to save to with --output_file')
if FLAGS.is_video_model and not FLAGS.num_frames:
raise ValueError(
'Number of frames must be specified for video models with --num_frames'
)
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as graph:
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name,
'train',
FLAGS.dataset_dir,
perc_validation=FLAGS.perc_validation,
labels_filename=FLAGS.labels_filename)
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
is_training=FLAGS.is_training)
image_size = FLAGS.image_size or network_fn.default_image_size
num_channels = 1 if FLAGS.use_grayscale else 3
if FLAGS.is_video_model:
input_shape = [
FLAGS.batch_size, FLAGS.num_frames, image_size, image_size,
num_channels
]
else:
input_shape = [
FLAGS.batch_size, image_size, image_size, num_channels
]
placeholder = tf.placeholder(name='input',
dtype=tf.float32,
shape=input_shape)
network_fn(placeholder)
if FLAGS.quantize:
contrib_quantize.create_eval_graph()
graph_def = graph.as_graph_def()
if FLAGS.write_text_graphdef:
tf.io.write_graph(graph_def,
os.path.dirname(FLAGS.output_file),
os.path.basename(FLAGS.output_file),
as_text=True)
else:
with gfile.GFile(FLAGS.output_file, 'wb') as f:
f.write(graph_def.SerializeToString())
def export_inference_graph(args):
if not args.train_dir:
raise ValueError(
'You must supply the path to save to with --train_dir')
if args.is_video_model and not args.num_frames:
raise ValueError(
'Number of frames must be specified for video models with --num_frames'
)
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as graph:
dataset = dataset_factory.get_dataset(args.dataset_name, 'train',
args.dataset_dir,
args.train_num_data)
network_fn = nets_factory.get_network_fn(
args.model_name,
num_classes=(dataset.num_classes - args.labels_offset),
is_training=args.is_training)
image_size = args.image_size or network_fn.default_image_size
num_channels = 1 if args.use_grayscale else 3
if args.is_video_model:
input_shape = [
args.batch_size, args.num_frames, image_size, image_size,
num_channels
]
else:
input_shape = [
args.batch_size, image_size, image_size, num_channels
]
placeholder = tf.placeholder(name='input',
dtype=tf.float32,
shape=input_shape)
network_fn(placeholder)
if args.quantize:
contrib_quantize.create_eval_graph()
graph_def = graph.as_graph_def()
if args.write_text_graphdef:
tf.io.write_graph(graph_def,
os.path.dirname(args.train_dir +
'/inference_graph.pb'),
os.path.basename(args.train_dir),
as_text=True)
else:
with gfile.GFile(args.train_dir + '/inference_graph.pb',
'wb') as f:
f.write(graph_def.SerializeToString())
def export_inference_graph(dataset_name, dataset_dir, model_name,
labels_offset, is_training, final_endpoint,
image_size, use_grayscale, is_video_model,
batch_size, num_frames, quantize,
write_text_graphdef, output_file):
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as graph:
dataset = dataset_factory.get_dataset(dataset_name, 'train',
dataset_dir)
network_fn = nets_factory.get_network_fn(
model_name,
num_classes=(dataset.num_classes - labels_offset),
is_training=is_training,
final_endpoint=final_endpoint)
image_size = image_size or network_fn.default_image_size
num_channels = 1 if use_grayscale else 3
if is_video_model:
input_shape = [
batch_size, num_frames, image_size, image_size, num_channels
]
else:
input_shape = [batch_size, image_size, image_size, num_channels]
placeholder = tf.placeholder(name='input',
dtype=tf.float32,
shape=input_shape)
network_fn(placeholder)
if quantize:
contrib_quantize.create_eval_graph()
graph_def = graph.as_graph_def()
if write_text_graphdef:
tf.io.write_graph(graph_def,
os.path.dirname(output_file),
os.path.basename(output_file),
as_text=True)
else:
with gfile.GFile(output_file, 'wb') as f:
f.write(graph_def.SerializeToString())
def model_fn(features, labels, mode, params, tf_sess=False):
"""
Create the model for estimator api
Args:
features: if input_layout == 'nhwc', a tensor with shape:
[BATCH_SIZE, go.N, go.N, get_features_planes()]
else, a tensor with shape:
[BATCH_SIZE, get_features_planes(), go.N, go.N]
labels: dict from string to tensor with shape
'pi_tensor': [BATCH_SIZE, go.N * go.N + 1]
'value_tensor': [BATCH_SIZE]
mode: a tf.estimator.ModeKeys (batchnorm params update for TRAIN only)
params: A dictionary (Typically derived from the FLAGS object.)
Returns: tf.estimator.EstimatorSpec with props
mode: same as mode arg
predictions: dict of tensors
'policy': [BATCH_SIZE, go.N * go.N + 1]
'value': [BATCH_SIZE]
loss: a single value tensor
train_op: train op
eval_metric_ops
return dict of tensors
logits: [BATCH_SIZE, go.N * go.N + 1]
"""
policy_output, value_output, logits = model_inference_fn(
features, mode == tf.estimator.ModeKeys.TRAIN, params)
# train ops
policy_cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits,
labels=tf.stop_gradient(
labels['pi_tensor'])))
value_cost = params['value_cost_weight'] * tf.reduce_mean(
tf.square(value_output - labels['value_tensor']))
reg_vars = [
v for v in tf.trainable_variables()
if 'bias' not in v.name and 'beta' not in v.name
]
l2_cost = params['l2_strength'] * \
tf.add_n([tf.nn.l2_loss(v) for v in reg_vars])
combined_cost = policy_cost + value_cost + l2_cost
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.train.piecewise_constant(global_step,
params['lr_boundaries'],
params['lr_rates'])
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# Insert quantization ops if requested
if params['quantize']:
if mode == tf.estimator.ModeKeys.TRAIN:
contrib_quantize.create_training_graph(
quant_delay=params['quant_delay'])
else:
contrib_quantize.create_eval_graph()
optimizer = tf.train.MomentumOptimizer(learning_rate,
params['sgd_momentum'])
# hvd multigpu
optimizer = hvd.DistributedOptimizer(optimizer)
if params['use_tpu']:
optimizer = contrib_tpu_python_tpu_tpu_optimizer.CrossShardOptimizer(
optimizer)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(combined_cost, global_step=global_step)
# return train_op for sess
if tf_sess: return train_op
# Computations to be executed on CPU, outside of the main TPU queues.
def eval_metrics_host_call_fn(policy_output,
value_output,
pi_tensor,
value_tensor,
policy_cost,
value_cost,
l2_cost,
combined_cost,
step,
est_mode=tf.estimator.ModeKeys.TRAIN):
policy_entropy = -tf.reduce_mean(
tf.reduce_sum(policy_output * tf.log(policy_output), axis=1))
# pi_tensor is one_hot when generated from sgfs (for supervised learning)
# and soft-max when using self-play records. argmax normalizes the two.
policy_target_top_1 = tf.argmax(pi_tensor, axis=1)
policy_output_in_top1 = tf.to_float(
tf.nn.in_top_k(policy_output, policy_target_top_1, k=1))
policy_output_in_top3 = tf.to_float(
tf.nn.in_top_k(policy_output, policy_target_top_1, k=3))
policy_top_1_confidence = tf.reduce_max(policy_output, axis=1)
policy_target_top_1_confidence = tf.boolean_mask(
policy_output,
tf.one_hot(policy_target_top_1,
tf.shape(policy_output)[1]))
value_cost_normalized = value_cost / params['value_cost_weight']
avg_value_observed = tf.reduce_mean(value_tensor)
with tf.variable_scope('metrics'):
metric_ops = {
'policy_cost':
tf.metrics.mean(policy_cost),
'value_cost':
tf.metrics.mean(value_cost),
'value_cost_normalized':
tf.metrics.mean(value_cost_normalized),
'l2_cost':
tf.metrics.mean(l2_cost),
'policy_entropy':
tf.metrics.mean(policy_entropy),
'combined_cost':
tf.metrics.mean(combined_cost),
'avg_value_observed':
tf.metrics.mean(avg_value_observed),
'policy_accuracy_top_1':
tf.metrics.mean(policy_output_in_top1),
'policy_accuracy_top_3':
tf.metrics.mean(policy_output_in_top3),
'policy_top_1_confidence':
tf.metrics.mean(policy_top_1_confidence),
'policy_target_top_1_confidence':
tf.metrics.mean(policy_target_top_1_confidence),
'value_confidence':
tf.metrics.mean(tf.abs(value_output)),
}
if est_mode == tf.estimator.ModeKeys.EVAL:
return metric_ops
# NOTE: global_step is rounded to a multiple of FLAGS.summary_steps.
eval_step = tf.reduce_min(step)
# Create summary ops so that they show up in SUMMARIES collection
# That way, they get logged automatically during training
summary_writer = contrib_summary.create_file_writer(FLAGS.work_dir)
with summary_writer.as_default(), \
contrib_summary.record_summaries_every_n_global_steps(
params['summary_steps'], eval_step):
for metric_name, metric_op in metric_ops.items():
contrib_summary.scalar(metric_name,
metric_op[1],
step=eval_step)
# Reset metrics occasionally so that they are mean of recent batches.
reset_op = tf.variables_initializer(tf.local_variables('metrics'))
cond_reset_op = tf.cond(
tf.equal(eval_step % params['summary_steps'], tf.to_int64(1)),
lambda: reset_op, lambda: tf.no_op())
return contrib_summary.all_summary_ops() + [cond_reset_op]
metric_args = [
policy_output,
value_output,
labels['pi_tensor'],
labels['value_tensor'],
tf.reshape(policy_cost, [1]),
tf.reshape(value_cost, [1]),
tf.reshape(l2_cost, [1]),
tf.reshape(combined_cost, [1]),
tf.reshape(global_step, [1]),
]
predictions = {
'policy_output': policy_output,
'value_output': value_output,
}
eval_metrics_only_fn = functools.partial(
eval_metrics_host_call_fn, est_mode=tf.estimator.ModeKeys.EVAL)
host_call_fn = functools.partial(eval_metrics_host_call_fn,
est_mode=tf.estimator.ModeKeys.TRAIN)
tpu_estimator_spec = contrib_tpu_python_tpu_tpu_estimator.TPUEstimatorSpec(
mode=mode,
predictions=predictions,
loss=combined_cost,
train_op=train_op)
if params['use_tpu']:
return tpu_estimator_spec
else:
return tpu_estimator_spec.as_estimator_spec()
def build_loss(self):
response = self.response
response_size = response.get_shape().as_list()[1:3] # [height, width]
gt = construct_gt_score_maps(
response_size, self.data_config['batch_size'],
self.model_config['embed_config']['stride'],
self.train_config['gt_config'])
# loss: https://www.renom.jp/ja/notebooks/tutorial/basic_algorithm/lossfunction/notebook.html
with tf.name_scope('Loss'):
loss = tf.nn.sigmoid_cross_entropy_with_logits(logits=response,
labels=gt)
with tf.name_scope('Balance_weights'):
n_pos = tf.reduce_sum(tf.to_float(tf.equal(gt[0], 1)))
n_neg = tf.reduce_sum(tf.to_float(tf.equal(gt[0], 0)))
w_pos = 0.5 / n_pos
w_neg = 0.5 / n_neg
class_weights = tf.where(tf.equal(gt, 1),
w_pos * tf.ones_like(gt),
tf.ones_like(gt))
class_weights = tf.where(tf.equal(gt, 0),
w_neg * tf.ones_like(gt),
class_weights)
loss = loss * class_weights
# Note that we use reduce_sum instead of reduce_mean since the loss has
# already been normalized by class_weights in spatial dimension.
loss = tf.reduce_sum(loss, [1, 2])
batch_loss = tf.reduce_mean(loss, name='batch_loss')
tf.losses.add_loss(batch_loss)
total_loss = tf.losses.get_total_loss()
self.batch_loss = batch_loss
self.total_loss = total_loss
# quantization
# good note: https://www.tensorflowers.cn/t/7136
if self.model_config['embed_config']['quantization']:
if self.train_config["export"]:
contrib_quantize.create_eval_graph()
else:
contrib_quantize.create_training_graph(quant_delay=200000)
tf.summary.image('exemplar', self.exemplars, family=self.mode)
tf.summary.image('instance', self.instances, family=self.mode)
mean_batch_loss, update_op1 = tf.metrics.mean(batch_loss)
mean_total_loss, update_op2 = tf.metrics.mean(total_loss)
with tf.control_dependencies([update_op1, update_op2]):
tf.summary.scalar('batch_loss',
mean_batch_loss,
family=self.mode)
tf.summary.scalar('total_loss',
mean_total_loss,
family=self.mode)
if self.mode == 'train':
tf.summary.image('GT',
tf.reshape(gt[0], [1] + response_size + [1]),
family='GT')
tf.summary.image('Response',
tf.expand_dims(tf.sigmoid(response), -1),
family=self.mode)
tf.summary.histogram('Response', self.response, family=self.mode)
# Two more metrics to monitor the performance of training
tf.summary.scalar('center_score_error',
center_score_error(response),
family=self.mode)
tf.summary.scalar('center_dist_error',
center_dist_error(response),
family=self.mode)
def build_model_fn(features, labels, mode, params):
"""The model_fn for MnasNet to be used with TPUEstimator.
Args:
features: `Tensor` of batched images.
labels: `Tensor` of labels for the data samples
mode: one of `tf.estimator.ModeKeys.{TRAIN,EVAL,PREDICT}`
params: `dict` of parameters passed to the model from the TPUEstimator,
`params['batch_size']` is always provided and should be used as the
effective batch size.
Returns:
A `TPUEstimatorSpec` for the model
"""
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# This is essential, if using a keras-derived model.
tf.keras.backend.set_learning_phase(is_training)
if isinstance(features, dict):
features = features['feature']
if mode == tf.estimator.ModeKeys.PREDICT:
# Adds an identify node to help TFLite export.
features = tf.identity(features, 'float_image_input')
# In most cases, the default data format NCHW instead of NHWC should be
# used for a significant performance boost on GPU. NHWC should be used
# only if the network needs to be run on CPU since the pooling operations
# are only supported on NHWC. TPU uses XLA compiler to figure out best layout.
if params['data_format'] == 'channels_first':
assert not params['transpose_input'] # channels_first only for GPU
features = tf.transpose(features, [0, 3, 1, 2])
stats_shape = [3, 1, 1]
else:
stats_shape = [1, 1, 3]
if params['transpose_input'] and mode != tf.estimator.ModeKeys.PREDICT:
features = tf.transpose(features, [3, 0, 1, 2]) # HWCN to NHWC
# Normalize the image to zero mean and unit variance.
features -= tf.constant(
imagenet_input.MEAN_RGB, shape=stats_shape, dtype=features.dtype)
features /= tf.constant(
imagenet_input.STDDEV_RGB, shape=stats_shape, dtype=features.dtype)
has_moving_average_decay = (params['moving_average_decay'] > 0)
tf.logging.info('Using open-source implementation for MnasNet definition.')
override_params = {}
if params['batch_norm_momentum']:
override_params['batch_norm_momentum'] = params['batch_norm_momentum']
if params['batch_norm_epsilon']:
override_params['batch_norm_epsilon'] = params['batch_norm_epsilon']
if params['dropout_rate']:
override_params['dropout_rate'] = params['dropout_rate']
if params['data_format']:
override_params['data_format'] = params['data_format']
if params['num_label_classes']:
override_params['num_classes'] = params['num_label_classes']
if params['depth_multiplier']:
override_params['depth_multiplier'] = params['depth_multiplier']
if params['depth_divisor']:
override_params['depth_divisor'] = params['depth_divisor']
if params['min_depth']:
override_params['min_depth'] = params['min_depth']
override_params['use_keras'] = params['use_keras']
def _build_model(model_name):
"""Build the model for a given model name."""
if model_name.startswith('mnasnet'):
return mnasnet_models.build_mnasnet_model(
features,
model_name=model_name,
training=is_training,
override_params=override_params)
elif model_name.startswith('mixnet'):
return mixnet_builder.build_model(
features,
model_name=model_name,
training=is_training,
override_params=override_params)
else:
raise ValueError('Unknown model name {}'.format(model_name))
if params['precision'] == 'bfloat16':
with tf.tpu.bfloat16_scope():
logits, _ = _build_model(params['model_name'])
logits = tf.cast(logits, tf.float32)
else: # params['precision'] == 'float32'
logits, _ = _build_model(params['model_name'])
if params['quantized_training']:
try:
from tensorflow.contrib import quantize # pylint: disable=g-import-not-at-top
except ImportError as e:
logging.exception('Quantized training is not supported in TensorFlow 2.x')
raise e
if is_training:
tf.logging.info('Adding fake quantization ops for training.')
quantize.create_training_graph(
quant_delay=int(params['steps_per_epoch'] *
FLAGS.quantization_delay_epochs))
else:
tf.logging.info('Adding fake quantization ops for evaluation.')
quantize.create_eval_graph()
if mode == tf.estimator.ModeKeys.PREDICT:
scaffold_fn = None
if FLAGS.export_moving_average:
# If the model is trained with moving average decay, to match evaluation
# metrics, we need to export the model using moving average variables.
restore_checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
variables_to_restore = get_pretrained_variables_to_restore(
restore_checkpoint, load_moving_average=True)
tf.logging.info('Restoring from the latest checkpoint: %s',
restore_checkpoint)
tf.logging.info(str(variables_to_restore))
def restore_scaffold():
saver = tf.train.Saver(variables_to_restore)
return tf.train.Scaffold(saver=saver)
scaffold_fn = restore_scaffold
predictions = {
'classes': tf.argmax(logits, axis=1),
'probabilities': tf.nn.softmax(logits, name='softmax_tensor')
}
return tf.estimator.tpu.TPUEstimatorSpec(
mode=mode,
predictions=predictions,
export_outputs={
'classify': tf.estimator.export.PredictOutput(predictions)
},
scaffold_fn=scaffold_fn)
# If necessary, in the model_fn, use params['batch_size'] instead the batch
# size flags (--train_batch_size or --eval_batch_size).
batch_size = params['batch_size'] # pylint: disable=unused-variable
# Calculate loss, which includes softmax cross entropy and L2 regularization.
one_hot_labels = tf.one_hot(labels, params['num_label_classes'])
cross_entropy = tf.losses.softmax_cross_entropy(
logits=logits,
onehot_labels=one_hot_labels,
label_smoothing=params['label_smoothing'])
# Add weight decay to the loss for non-batch-normalization variables.
loss = cross_entropy + params['weight_decay'] * tf.add_n([
tf.nn.l2_loss(v)
for v in tf.trainable_variables()
if 'batch_normalization' not in v.name
])
global_step = tf.train.get_global_step()
if has_moving_average_decay:
ema = tf.train.ExponentialMovingAverage(
decay=params['moving_average_decay'], num_updates=global_step)
ema_vars = mnas_utils.get_ema_vars()
host_call = None
if is_training:
# Compute the current epoch and associated learning rate from global_step.
current_epoch = (
tf.cast(global_step, tf.float32) / params['steps_per_epoch'])
scaled_lr = params['base_learning_rate'] * (params['train_batch_size'] / 256.0) # pylint: disable=line-too-long
learning_rate = mnas_utils.build_learning_rate(scaled_lr, global_step,
params['steps_per_epoch'])
optimizer = mnas_utils.build_optimizer(learning_rate)
if params['use_tpu']:
# When using TPU, wrap the optimizer with CrossShardOptimizer which
# handles synchronization details between different TPU cores. To the
# user, this should look like regular synchronous training.
optimizer = tf.tpu.CrossShardOptimizer(optimizer)
if params['add_summaries']:
summary_writer = tf2.summary.create_file_writer(
FLAGS.model_dir, max_queue=params['iterations_per_loop'])
with summary_writer.as_default():
should_record = tf.equal(global_step % params['iterations_per_loop'],
0)
with tf2.summary.record_if(should_record):
tf2.summary.scalar('loss', loss, step=global_step)
tf2.summary.scalar('learning_rate', learning_rate, step=global_step)
tf2.summary.scalar('current_epoch', current_epoch, step=global_step)
# Batch normalization requires UPDATE_OPS to be added as a dependency to
# the train operation.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops + tf.summary.all_v2_summary_ops()):
train_op = optimizer.minimize(loss, global_step)
if has_moving_average_decay:
with tf.control_dependencies([train_op]):
train_op = ema.apply(ema_vars)
else:
train_op = None
eval_metrics = None
if mode == tf.estimator.ModeKeys.EVAL:
def metric_fn(labels, logits):
"""Evaluation metric function.
Evaluates accuracy.
This function is executed on the CPU and should not directly reference
any Tensors in the rest of the `model_fn`. To pass Tensors from the model
to the `metric_fn`, provide as part of the `eval_metrics`. See
https://www.tensorflow.org/api_docs/python/tf/estimator/tpu/TPUEstimatorSpec
for more information.
Arguments should match the list of `Tensor` objects passed as the second
element in the tuple passed to `eval_metrics`.
Args:
labels: `Tensor` with shape `[batch]`.
logits: `Tensor` with shape `[batch, num_classes]`.
Returns:
A dict of the metrics to return from evaluation.
"""
predictions = tf.argmax(logits, axis=1)
top_1_accuracy = tf.metrics.accuracy(labels, predictions)
in_top_5 = tf.cast(tf.nn.in_top_k(logits, labels, 5), tf.float32)
top_5_accuracy = tf.metrics.mean(in_top_5)
return {
'top_1_accuracy': top_1_accuracy,
'top_5_accuracy': top_5_accuracy,
}
eval_metrics = (metric_fn, [labels, logits])
num_params = np.sum([np.prod(v.shape) for v in tf.trainable_variables()])
tf.logging.info('number of trainable parameters: {}'.format(num_params))
# Prepares scaffold_fn if needed.
scaffold_fn = None
if is_training and FLAGS.init_checkpoint:
variables_to_restore = get_pretrained_variables_to_restore(
FLAGS.init_checkpoint, has_moving_average_decay)
tf.logging.info('Initializing from pretrained checkpoint: %s',
FLAGS.init_checkpoint)
if FLAGS.use_tpu:
def init_scaffold():
tf.train.init_from_checkpoint(FLAGS.init_checkpoint,
variables_to_restore)
return tf.train.Scaffold()
scaffold_fn = init_scaffold
else:
tf.train.init_from_checkpoint(FLAGS.init_checkpoint, variables_to_restore)
restore_vars_dict = None
if not is_training and has_moving_average_decay:
# Load moving average variables for eval.
restore_vars_dict = ema.variables_to_restore(ema_vars)
def eval_scaffold():
saver = tf.train.Saver(restore_vars_dict)
return tf.train.Scaffold(saver=saver)
scaffold_fn = eval_scaffold
return tf.estimator.tpu.TPUEstimatorSpec(
mode=mode,
loss=loss,
train_op=train_op,
host_call=host_call,
eval_metrics=eval_metrics,
scaffold_fn=scaffold_fn)
def main():
# check required input arguments
if not FLAGS.project_name:
raise ValueError('You must supply a project name with --project_name')
if not FLAGS.dataset_name:
raise ValueError('You must supply a dataset name with --dataset_name')
# set and check project_dir and experiment_dir.
project_dir = os.path.join(FLAGS.project_dir, FLAGS.project_name)
if not FLAGS.experiment_name:
# list only directories that are names experiment_
experiment_dir = dataset_utils.select_latest_experiment_dir(
project_dir)
else:
experiment_dir = os.path.join(os.path.join(project_dir, 'experiments'),
FLAGS.experiment_name)
if not os.path.exists(experiment_dir):
raise ValueError('Experiment directory {} does not exist.'.format(
experiment_dir))
eval_dir = os.path.join(experiment_dir, FLAGS.dataset_split_name)
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
if FLAGS.dataset_dir:
dataset_dir = os.path.join(FLAGS.dataset_dir, FLAGS.dataset_name)
else:
dataset_dir = os.path.join(os.path.join(project_dir, 'datasets'),
FLAGS.dataset_name)
if not os.path.isdir(dataset_dir):
raise ValueError(
'Can not find tfrecord dataset directory {}'.format(dataset_dir))
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# tf_global_step = slim.get_or_create_global_step()
tf_global_step = tf.train.get_or_create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
dataset_dir)
####################
# Select the model #
####################
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
is_training=False,
final_endpoint=FLAGS.final_endpoint)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
num_epochs=None,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size)
[image, label] = provider.get(['image', 'label'])
label -= FLAGS.labels_offset
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name,
is_training=False,
use_grayscale=FLAGS.use_grayscale)
eval_image_size = FLAGS.eval_image_size or network_fn.default_image_size
image = image_preprocessing_fn(
image,
eval_image_size,
eval_image_size,
add_image_summaries=FLAGS.add_image_summaries,
roi=_parse_roi())
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
####################
# Define the model #
####################
logits, _ = network_fn(images)
if FLAGS.quantize:
contrib_quantize.create_eval_graph()
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
loss = tf.losses.softmax_cross_entropy(
tf.one_hot(labels, dataset.num_classes), logits)
#############################
## Calculation of metrics ##
#############################
accuracy, accuracy_op = tf.metrics.accuracy(tf.squeeze(labels),
tf.argmax(logits, 1))
precision, precision_op = tf.metrics.average_precision_at_k(
tf.squeeze(labels), logits, 1)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_ops.append([accuracy_op, precision_op])
tf.add_to_collection('accuracy', accuracy)
tf.add_to_collection('accuracy_op', accuracy_op)
tf.add_to_collection('precision', precision)
tf.add_to_collection('precision_op', precision_op)
for class_id in range(dataset.num_classes):
precision_at_k, precision_at_k_op = tf.metrics.precision_at_k(
tf.squeeze(labels), logits, k=1, class_id=class_id)
recall_at_k, recall_at_k_op = tf.metrics.recall_at_k(
tf.squeeze(labels), logits, k=1, class_id=class_id)
update_ops.append([precision_at_k_op, recall_at_k_op])
tf.add_to_collection('precision_at_{}'.format(class_id),
precision_at_k)
tf.add_to_collection('precision_at_{}_op'.format(class_id),
precision_at_k_op)
tf.add_to_collection('recall_at_{}'.format(class_id), recall_at_k)
tf.add_to_collection('recall_at_{}_op'.format(class_id),
recall_at_k_op)
#############################
## Add summaries ##
#############################
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# Add summaries for losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss))
summaries.add(tf.summary.scalar('Losses/clone_loss', loss))
summaries.add(tf.summary.scalar('Metrics/accuracy', accuracy))
summaries.add(tf.summary.scalar('op/accuracy_op', accuracy_op))
summaries.add(tf.summary.scalar('Metrics/average_precision',
precision))
summaries.add(
tf.summary.scalar('op/average_precision_op', precision_op))
for class_id in range(dataset.num_classes):
precision_at_k = tf.get_collection(
'precision_at_{}'.format(class_id))
precision_at_k_op = tf.get_collection(
'precision_at_{}_op'.format(class_id))
recall_at_k = tf.get_collection('recall_at_{}'.format(class_id))
recall_at_k_op = tf.get_collection(
'recall_at_{}_op'.format(class_id))
precision_at_k = tf.reshape(precision_at_k, [])
precision_at_k_op = tf.reshape(precision_at_k_op, [])
recall_at_k = tf.reshape(recall_at_k, [])
recall_at_k_op = tf.reshape(recall_at_k_op, [])
summaries.add(
tf.summary.scalar(
'Metrics/class_{}_precision'.format(class_id),
precision_at_k))
summaries.add(
tf.summary.scalar('op/class_{}_precision_op'.format(class_id),
precision_at_k_op))
summaries.add(
tf.summary.scalar('Metrics/class_{}_recall'.format(class_id),
recall_at_k))
summaries.add(
tf.summary.scalar('op/class_{}_recall_op'.format(class_id),
recall_at_k_op))
# set batch size if none to
# number_of_samples_in_dataset / batch_size
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
# This ensures that we make a single pass over all of the data.
num_batches = math.ceil(dataset.num_samples /
float(FLAGS.batch_size))
# if checkpoint_path flag is none, look for checkpoint in experiment train directory
if FLAGS.checkpoint_path is None:
checkpoint_path = os.path.join(experiment_dir, 'train')
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating checkpoint: %s' % checkpoint_path)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
total_loss = tf.identity(loss, name='total_loss')
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# configure session
session_config = tf.ConfigProto(
log_device_placement=FLAGS.verbose_placement,
allow_soft_placement=not FLAGS.hard_placement)
if not FLAGS.fixed_memory:
session_config.gpu_options.allow_growth = True
# set evaluation interval
if not FLAGS.eval_timeout_secs:
eval_timeout_secs = FLAGS.eval_interval_secs * 2
else:
eval_timeout_secs = FLAGS.eval_timeout_secs
# Evaluate every 1 minute:
slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=checkpoint_path,
logdir=eval_dir,
num_evals=num_batches,
eval_op=update_ops,
summary_op=summary_op,
eval_interval_secs=FLAGS.eval_interval_secs,
timeout=eval_timeout_secs,
session_config=session_config)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
# Get dataset-dependent information.
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.vis_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.vis_batch_size,
crop_size=[int(sz) for sz in FLAGS.vis_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
is_training=False,
should_shuffle=False,
should_repeat=False)
train_id_to_eval_id = None
if dataset.dataset_name == data_generator.get_cityscapes_dataset_name():
tf.logging.info('Cityscapes requires converting train_id to eval_id.')
train_id_to_eval_id = _CITYSCAPES_TRAIN_ID_TO_EVAL_ID
# Prepare for visualization.
tf.gfile.MakeDirs(FLAGS.vis_logdir)
save_dir = os.path.join(FLAGS.vis_logdir, _SEMANTIC_PREDICTION_SAVE_FOLDER)
tf.gfile.MakeDirs(save_dir)
raw_save_dir = os.path.join(
FLAGS.vis_logdir, _RAW_SEMANTIC_PREDICTION_SAVE_FOLDER)
tf.gfile.MakeDirs(raw_save_dir)
tf.logging.info('Visualizing on %s set', FLAGS.vis_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={common.OUTPUT_TYPE: dataset.num_of_classes},
crop_size=[int(sz) for sz in FLAGS.vis_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
if tuple(FLAGS.eval_scales) == (1.0,):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options=model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
if FLAGS.min_resize_value and FLAGS.max_resize_value:
# Only support batch_size = 1, since we assume the dimensions of original
# image after tf.squeeze is [height, width, 3].
assert FLAGS.vis_batch_size == 1
# Reverse the resizing and padding operations performed in preprocessing.
# First, we slice the valid regions (i.e., remove padded region) and then
# we resize the predictions back.
original_image = tf.squeeze(samples[common.ORIGINAL_IMAGE])
original_image_shape = tf.shape(original_image)
predictions = tf.slice(
predictions,
[0, 0, 0],
[1, original_image_shape[0], original_image_shape[1]])
resized_shape = tf.to_int32([tf.squeeze(samples[common.HEIGHT]),
tf.squeeze(samples[common.WIDTH])])
predictions = tf.squeeze(
tf.image.resize_images(tf.expand_dims(predictions, 3),
resized_shape,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
align_corners=True), 3)
tf.train.get_or_create_global_step()
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
num_iteration = 0
max_num_iteration = FLAGS.max_number_of_iterations
checkpoints_iterator = contrib_training.checkpoints_iterator(
FLAGS.checkpoint_dir, min_interval_secs=FLAGS.eval_interval_secs)
for checkpoint_path in checkpoints_iterator:
num_iteration += 1
tf.logging.info(
'Starting visualization at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
time.gmtime()))
tf.logging.info('Visualizing with model %s', checkpoint_path)
scaffold = tf.train.Scaffold(init_op=tf.global_variables_initializer())
session_creator = tf.train.ChiefSessionCreator(
scaffold=scaffold,
master=FLAGS.master,
checkpoint_filename_with_path=checkpoint_path)
with tf.train.MonitoredSession(
session_creator=session_creator, hooks=None) as sess:
batch = 0
image_id_offset = 0
while not sess.should_stop():
tf.logging.info('Visualizing batch %d', batch + 1)
_process_batch(sess=sess,
original_images=samples[common.ORIGINAL_IMAGE],
semantic_predictions=predictions,
image_names=samples[common.IMAGE_NAME],
image_heights=samples[common.HEIGHT],
image_widths=samples[common.WIDTH],
image_id_offset=image_id_offset,
save_dir=save_dir,
raw_save_dir=raw_save_dir,
train_id_to_eval_id=train_id_to_eval_id)
image_id_offset += FLAGS.vis_batch_size
batch += 1
tf.logging.info(
'Finished visualization at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
time.gmtime()))
if max_num_iteration > 0 and num_iteration >= max_num_iteration:
break
def add_final_retrain_ops(class_count, final_tensor_name, bottleneck_tensor,
quantize_layer, is_training):
"""Adds a new softmax and fully-connected layer for training and eval.
We need to retrain the top layer to identify our new classes, so this function
adds the right operations to the graph, along with some variables to hold the
weights, and then sets up all the gradients for the backward pass.
The set up for the softmax and fully-connected layers is based on:
https://www.tensorflow.org/tutorials/mnist/beginners/index.html
Args:
class_count: Integer of how many categories of things we're trying to
recognize.
final_tensor_name: Name string for the new final node that produces results.
bottleneck_tensor: The output of the main CNN graph.
quantize_layer: Boolean, specifying whether the newly added layer should be
instrumented for quantization with TF-Lite.
is_training: Boolean, specifying whether the newly add layer is for training
or eval.
Returns:
The tensors for the training and cross entropy results, and tensors for the
bottleneck input and ground truth input.
"""
batch_size, bottleneck_tensor_size = bottleneck_tensor.get_shape().as_list()
assert batch_size is None, 'We want to work with arbitrary batch size.'
with tf.name_scope('input'):
bottleneck_input = tf.placeholder_with_default(
bottleneck_tensor,
shape=[batch_size, bottleneck_tensor_size],
name='BottleneckInputPlaceholder')
ground_truth_input = tf.placeholder(
tf.int64, [batch_size], name='GroundTruthInput')
# Organizing the following ops so they are easier to see in TensorBoard.
layer_name = 'final_retrain_ops'
with tf.name_scope(layer_name):
with tf.name_scope('weights'):
initial_value = tf.truncated_normal(
[bottleneck_tensor_size, class_count], stddev=0.001)
layer_weights = tf.Variable(initial_value, name='final_weights')
variable_summaries(layer_weights)
with tf.name_scope('biases'):
layer_biases = tf.Variable(tf.zeros([class_count]), name='final_biases')
variable_summaries(layer_biases)
with tf.name_scope('Wx_plus_b'):
logits = tf.matmul(bottleneck_input, layer_weights) + layer_biases
tf.summary.histogram('pre_activations', logits)
final_tensor = tf.nn.softmax(logits, name=final_tensor_name)
# The tf.contrib.quantize functions rewrite the graph in place for
# quantization. The imported model graph has already been rewritten, so upon
# calling these rewrites, only the newly added final layer will be
# transformed.
if quantize_layer:
if is_training:
contrib_quantize.create_training_graph()
else:
contrib_quantize.create_eval_graph()
tf.summary.histogram('activations', final_tensor)
# If this is an eval graph, we don't need to add loss ops or an optimizer.
if not is_training:
return None, None, bottleneck_input, ground_truth_input, final_tensor
with tf.name_scope('cross_entropy'):
cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
labels=ground_truth_input, logits=logits)
tf.summary.scalar('cross_entropy', cross_entropy_mean)
with tf.name_scope('train'):
optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)
train_step = optimizer.minimize(cross_entropy_mean)
return (train_step, cross_entropy_mean, bottleneck_input, ground_truth_input,
final_tensor)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
if not FLAGS.frozen_pb:
raise ValueError('You must supply the frozen pb with --frozen_pb')
if not FLAGS.output_node_name:
raise ValueError(
'You must supply the output node name with --output_node_name')
if not FLAGS.output_dir:
raise ValueError(
'You must supply the output directory with --output_dir')
tf.logging.set_verbosity(tf.logging.INFO)
tfrecords = prepare_tfrecords(FLAGS.dataset_name, FLAGS.dataset_dir,
FLAGS.dataset_split_name)
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
num_records = sum(
[len(list(tf.python_io.tf_record_iterator(r))) for r in tfrecords])
num_batches = int(math.ceil(num_records / float(FLAGS.batch_size)))
tf.logging.info('Load GraphDef from frozen_pb {}'.format(FLAGS.frozen_pb))
graph_def = load_graph_def(FLAGS.frozen_pb)
tf.logging.info('Quantize Graph')
with tf.Session() as sess:
tf.import_graph_def(graph_def, name='')
quantized_graph = qg.create_training_graph(sess.graph)
quantized_inf_graph = qg.create_eval_graph(sess.graph)
# Initialize `iterator` with training data.
with tf.Session(graph=quantized_graph) as sess:
tf.logging.info('Prepare dataset')
with tf.name_scope("dataset"):
filenames = tf.placeholder(tf.string, shape=[None])
dataset = prepare_dataset(filenames,
FLAGS.dataset_name,
FLAGS.input_size,
batch_size=FLAGS.batch_size)
iterator = dataset.make_initializable_iterator()
next_batch = iterator.get_next()
tf.logging.info('Prepare metrics')
lbls, preds, accuracy, acc_update_op = prepare_metrics(
FLAGS.dataset_name)
tf.logging.info('Prepare Saver')
saver = tf.train.Saver()
if FLAGS.summary_dir:
tf.logging.info('Prepare summary writer')
summary_writer = tf.summary.FileWriter(FLAGS.summary_dir)
# initialize
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(iterator.initializer, feed_dict={filenames: tfrecords})
graph = sess.graph
# get x and y
x = graph.get_tensor_by_name('{}:0'.format(FLAGS.input_node_name))
y = graph.get_tensor_by_name('{}:0'.format(FLAGS.output_node_name))
# summary all min/max variables
# print(graph.get_collection('variables')[3].eval())
for var in graph.get_collection('variables'):
tf.summary.scalar(var.name, var)
summaries = tf.summary.merge_all()
for step in range(num_batches):
images, labels = sess.run(next_batch)
ys = sess.run(y, feed_dict={x: images})
sess.run(acc_update_op, feed_dict={lbls: labels, preds: ys})
summary = sess.run(summaries)
if FLAGS.summary_dir:
summary_writer.add_summary(summary, step)
print('Accuracy: [{:.4f}]'.format(sess.run(accuracy)))
if FLAGS.summary_dir:
summary_writer.add_graph(graph)
# save graph and ckpts
saver.save(sess, os.path.join(FLAGS.output_dir, "model.ckpt"))
# tf.train.write_graph(graph, FLAGS.output_dir, 'quantor.pb', as_text=False)
tf.train.write_graph(quantized_inf_graph,
FLAGS.output_dir,
'quantor.pb',
as_text=False)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
# Get dataset-dependent information.
dataset = data_generator.Dataset( # 获取相应数据集
dataset_name=FLAGS.dataset, # 数据集名称
split_name=FLAGS.vis_split, # 用于语义分割的数据集的tfrecorder文件 默认带有val
dataset_dir=FLAGS.dataset_dir, # 数据集目录
batch_size=FLAGS.vis_batch_size, # 一次性处理的image_batch_size 默认为1
crop_size=[int(sz)
for sz in FLAGS.vis_crop_size], #crop_size 默认为513,513
min_resize_value=FLAGS.min_resize_value, # None
max_resize_value=FLAGS.max_resize_value, # None
resize_factor=FLAGS.resize_factor, # None
model_variant=FLAGS.
model_variant, # 模型的变体 默认为mobilenet_v2 本次训练为 xception_65
is_training=False, # 不训练
should_shuffle=False, # 不将输入的数据随机打乱
should_repeat=False) # 不一直重复
train_id_to_eval_id = None
if dataset.dataset_name == data_generator.get_cityscapes_dataset_name():
tf.logging.info('Cityscapes requires converting train_id to eval_id.')
train_id_to_eval_id = _CITYSCAPES_TRAIN_ID_TO_EVAL_ID
# Prepare for visualization.
tf.gfile.MakeDirs(FLAGS.vis_logdir) # 可视化图片放置的文件夹
save_dir = os.path.join(FLAGS.vis_logdir,
_SEMANTIC_PREDICTION_SAVE_FOLDER) # 创建存放文件夹
tf.gfile.MakeDirs(save_dir) # 创建segmentation_results文件夹
raw_save_dir = os.path.join(FLAGS.vis_logdir,
_RAW_SEMANTIC_PREDICTION_SAVE_FOLDER)
tf.gfile.MakeDirs(raw_save_dir) # 创建 raw_segmentation_results文件夹
tf.logging.info('Visualizing on %s set', FLAGS.vis_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next() # 获取数据
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.vis_crop_size], # 1024,2048
atrous_rates=FLAGS.atrous_rates, # 6,12,18
output_stride=FLAGS.output_stride) # 4
if tuple(FLAGS.eval_scales) == (1.0, ): # 不缩放进行评估
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels( # 标签预测 跟eval一样
samples[common.IMAGE],
model_options=model_options,
image_pyramid=FLAGS.image_pyramid)
else: # 多尺度评估
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
'''
predictions:
{'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>,
'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>}
'''
predictions = predictions[common.OUTPUT_TYPE]
if FLAGS.min_resize_value and FLAGS.max_resize_value: # None 暂不考虑
# Only support batch_size = 1, since we assume the dimensions of original
# image after tf.squeeze is [height, width, 3].
assert FLAGS.vis_batch_size == 1
# Reverse the resizing and padding operations performed in preprocessing.
# First, we slice the valid regions (i.e., remove padded region) and then
# we resize the predictions back.
original_image = tf.squeeze(samples[common.ORIGINAL_IMAGE])
original_image_shape = tf.shape(original_image)
predictions = tf.slice(
predictions, [0, 0, 0],
[1, original_image_shape[0], original_image_shape[1]])
resized_shape = tf.to_int32([
tf.squeeze(samples[common.HEIGHT]),
tf.squeeze(samples[common.WIDTH])
])
predictions = tf.squeeze(
tf.image.resize_images(
tf.expand_dims(predictions, 3),
resized_shape,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
align_corners=True), 3)
# 计数作用，每进行一个batch, global加１
tf.train.get_or_create_global_step()
if FLAGS.quantize_delay_step >= 0: # 默认为-1
contrib_quantize.create_eval_graph()
num_iteration = 0
max_num_iteration = FLAGS.max_number_of_iterations # 0
checkpoints_iterator = contrib_training.checkpoints_iterator(
FLAGS.checkpoint_dir, min_interval_secs=FLAGS.eval_interval_secs)
for checkpoint_path in checkpoints_iterator:
num_iteration += 1
tf.logging.info('Starting visualization at ' +
time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
tf.logging.info('Visualizing with model %s', checkpoint_path)
scaffold = tf.train.Scaffold(
init_op=tf.global_variables_initializer())
session_creator = tf.train.ChiefSessionCreator(
scaffold=scaffold,
master=FLAGS.master,
checkpoint_filename_with_path=checkpoint_path)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=None) as sess:
batch = 0
image_id_offset = 0
while not sess.should_stop():
tf.logging.info('Visualizing batch %d', batch + 1)
_process_batch(
sess=sess,
original_images=samples[
common.
ORIGINAL_IMAGE], # <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>
semantic_predictions=
predictions, # <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>
image_names=samples[
common.
IMAGE_NAME], # <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>
image_heights=samples[
common.
HEIGHT], # <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>
image_widths=samples[
common.
WIDTH], # <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>
image_id_offset=image_id_offset, # 0
save_dir=save_dir, # 语义分割结果放置的路径
raw_save_dir=raw_save_dir,
train_id_to_eval_id=train_id_to_eval_id
) # 只有cityscape中不为None
image_id_offset += FLAGS.vis_batch_size # 可视化的imageId
batch += 1
tf.logging.info('Finished visualization at ' +
time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
if max_num_iteration > 0 and num_iteration >= max_num_iteration:
break
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
# Get dataset-dependent information.
dataset = wsi_data_generator.Dataset(
dataset_name=FLAGS.dataset,
dataset_dir=FLAGS.dataset_dir,
num_of_classes=FLAGS.num_classes,
downsample=FLAGS.wsi_downsample,
overlap_num=FLAGS.overlap_num,
batch_size=FLAGS.vis_batch_size,
crop_size=FLAGS.vis_crop_size,
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
is_training=False,
should_shuffle=False,
should_repeat=False)
if os.path.isfile(FLAGS.dataset_dir):
slides = [FLAGS.dataset_dir]
else:
# get all WSI in test set
slides = dataset._get_all_files(with_xml=False)
for slide in slides:
print('Working on: [{}]'.format(slide))
# get slide size and create empty wsi mask
slide_size = get_slide_size(slide)
def get_downsampled_size(size, downsample=FLAGS.wsi_downsample):
size /= downsample
return int(np.ceil(size))
mask_size = [
get_downsampled_size(slide_size[0]),
get_downsampled_size(slide_size[1])
]
slide_mask = np.zeros([slide_size[1], slide_size[0]], dtype=np.uint8)
train_id_to_eval_id = None
raw_save_dir = None
if FLAGS.also_save_raw_predictions:
raw_save_dir = os.path.join(FLAGS.vis_logdir,
_RAW_SEMANTIC_PREDICTION_SAVE_FOLDER)
# Prepare for visualization.
tf.gfile.MakeDirs(FLAGS.vis_logdir)
tf.gfile.MakeDirs(raw_save_dir)
with tf.Graph().as_default():
iterator, num_samples = dataset.get_one_shot_iterator_grid(slide)
samples = iterator.get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[FLAGS.vis_crop_size, FLAGS.vis_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
tf.logging.info('Performing WSI patch detection.\n')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options=model_options,
image_pyramid=FLAGS.image_pyramid)
predictions = predictions[common.OUTPUT_TYPE]
tf.train.get_or_create_global_step()
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
# checkpoints_iterator = contrib_training.checkpoints_iterator(
# FLAGS.checkpoint_dir, min_interval_secs=FLAGS.eval_interval_secs)
# for checkpoint_path in checkpoints_iterator:
checkpoint_path = FLAGS.checkpoint_dir
# tf.logging.info(
# 'Starting visualization at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
# time.gmtime()))
# tf.logging.info('Visualizing with model %s', checkpoint_path)
scaffold = tf.train.Scaffold(
init_op=tf.global_variables_initializer())
session_creator = tf.train.ChiefSessionCreator(
scaffold=scaffold,
master=FLAGS.master,
checkpoint_filename_with_path=checkpoint_path)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=None) as sess:
batch = 0
image_id_offset = 0
while not sess.should_stop():
# tf.logging.info('Visualizing batch %d', batch + 1)
print('\rWorking on batch: [{} of {}]'.format(
batch, num_samples),
end='')
slide_mask = _process_batch(
sess=sess,
slide_mask=slide_mask,
original_images=samples[common.IMAGE],
semantic_predictions=predictions,
image_names=samples[common.IMAGE_NAME],
mask_size=mask_size,
downsample=FLAGS.wsi_downsample,
image_heights=samples[common.HEIGHT],
image_widths=samples[common.WIDTH],
image_id_offset=image_id_offset,
raw_save_dir=raw_save_dir,
train_id_to_eval_id=train_id_to_eval_id)
image_id_offset += FLAGS.vis_batch_size
batch += 1
# remove small objects
if FLAGS.min_size != None and FLAGS.min_size > 0:
min_pixel_size = FLAGS.min_size / FLAGS.wsi_downsample
print('\ncleaning up small objects < {} pixels'.format(
min_pixel_size))
for iter in range(FLAGS.num_classes)[1:]:
boolmask = slide_mask == iter
cleanMask = remove_small_objects(boolmask.astype(bool),
min_pixel_size)
slide_mask[slide_mask == iter] = 0
slide_mask += cleanMask
if FLAGS.save_json_annotation:
anot_filename = FLAGS.json_filename
print('\ncreating annotation file: [{}]'.format(anot_filename))
root = mask_to_xml(xml_path=anot_filename,
mask=slide_mask,
downsample=FLAGS.wsi_downsample,
return_root=True)
json_data = convert_xml_json(root, ['gloms'])
import json
with open(anot_filename, 'w') as annotation_file:
json.dump(json_data,
annotation_file,
indent=2,
sort_keys=False)
else:
anot_filename = '{}.xml'.format(slide.split('.')[0])
print('\ncreating annotation file: [{}]'.format(anot_filename))
mask_to_xml(xml_path=anot_filename,
mask=slide_mask,
downsample=FLAGS.wsi_downsample)
print('annotation file saved...\n\n')
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
####################
# Select the model #
####################
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
is_training=False)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size)
[image, label] = provider.get(['image', 'label'])
label -= FLAGS.labels_offset
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name,
is_training=False,
use_grayscale=FLAGS.use_grayscale)
eval_image_size = FLAGS.eval_image_size or network_fn.default_image_size
image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
####################
# Define the model #
####################
logits, _ = network_fn(images)
if FLAGS.quantize:
contrib_quantize.create_eval_graph()
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
predictions = tf.argmax(logits, 1)
labels = tf.squeeze(labels)
# Define the metrics:
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'Accuracy':
slim.metrics.streaming_accuracy(predictions, labels),
'Recall_5':
slim.metrics.streaming_recall_at_k(logits, labels, 5),
})
# Print the summaries to screen.
for name, value in names_to_values.items():
summary_name = 'eval/%s' % name
op = tf.compat.v1.summary.scalar(summary_name,
value,
collections=[])
op = tf.Print(op, [value], summary_name)
tf.add_to_collection(tf.compat.v1.GraphKeys.SUMMARIES, op)
# TODO(sguada) use num_epochs=1
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
# This ensures that we make a single pass over all of the data.
num_batches = math.ceil(dataset.num_samples /
float(FLAGS.batch_size))
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.compat.v1.logging.info('Evaluating %s' % checkpoint_path)
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes,
common.INSTANCE: 1,
common.OFFSET: 2
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions_semantic = predictions[common.OUTPUT_TYPE]
predictions_center_points = predictions[common.INSTANCE]
predictions_offset_vectors = predictions[common.OFFSET]
# tf Non-maxima Suppression
# Pooling based NMS for Pooling Instance Centers
# Filtering predictions that are less than 0.1
instance_prediction = generate_instance_segmentation(
predictions_semantic, predictions_center_points,
predictions_offset_vectors)
category_prediction = tf.squeeze(predictions_semantic)
category_label = tf.squeeze(samples[common.LABEL][0])
not_ignore_mask = tf.not_equal(category_label, 255)
category_label = tf.cast(
category_label * tf.cast(not_ignore_mask, tf.int32), tf.int32)
instance_label = tf.squeeze(samples[common.LABEL_INSTANCE_IDS][0])
category_prediction = category_prediction * tf.cast(
not_ignore_mask, tf.int64)
instance_prediction = instance_prediction * tf.cast(
not_ignore_mask, tf.int64)
# Define the evaluation metric.
metric_map = {}
metric_map[
'panoptic_quality'] = streaming_metrics.streaming_panoptic_quality(
category_label,
instance_label,
category_prediction,
instance_prediction,
num_classes=19,
max_instances_per_category=256,
ignored_label=255,
offset=256 * 256)
metric_map[
'parsing_covering'] = streaming_metrics.streaming_parsing_covering(
category_label,
instance_label,
category_prediction,
instance_prediction,
num_classes=19,
max_instances_per_category=256,
ignored_label=255,
offset=256 * 256,
normalize_by_image_size=True)
metrics_to_values, metrics_to_updates = slim.metrics.aggregate_metric_map(
metric_map)
summary_ops = []
for metric_name, metric_value in metrics_to_values.iteritems():
if metric_name == 'panoptic_quality':
[pq, sq, rq, total_tp, total_fn,
total_fp] = tf.unstack(metric_value, 6, axis=0)
panoptic_metrics = {
# Panoptic quality.
'pq': pq,
# Segmentation quality.
'sq': sq,
# Recognition quality.
'rq': rq,
# Total true positives.
'total_tp': total_tp,
# Total false negatives.
'total_fn': total_fn,
# Total false positives.
'total_fp': total_fp,
}
# Find the valid classes that will be used for evaluation. We will
# ignore the `ignore_label` class and other classes which have (tp + fn
# + fp) equal to 0.
valid_classes = tf.logical_and(
tf.not_equal(tf.range(0, dataset.num_of_classes),
dataset.ignore_label),
tf.not_equal(total_tp + total_fn + total_fp, 0))
for target_metric, target_value in panoptic_metrics.iteritems(
):
output_metric_name = '{}_{}'.format(
metric_name, target_metric)
op = tf.summary.scalar(
output_metric_name,
tf.reduce_mean(
tf.boolean_mask(target_value, valid_classes)))
op = tf.Print(op, [target_value],
output_metric_name + '_classwise: ',
summarize=dataset.num_of_classes)
op = tf.Print(op, [
tf.reduce_mean(
tf.boolean_mask(target_value, valid_classes))
],
output_metric_name + '_mean: ',
summarize=1)
summary_ops.append(op)
elif metric_name == 'parsing_covering':
[
per_class_covering, total_per_class_weighted_ious,
total_per_class_gt_areas
] = tf.unstack(metric_value, 3, axis=0)
# Find the valid classes that will be used for evaluation. We will
# ignore the `void_label` class and other classes which have
# total_per_class_weighted_ious + total_per_class_gt_areas equal to 0.
valid_classes = tf.logical_and(
tf.not_equal(tf.range(0, dataset.num_of_classes),
dataset.ignore_label),
tf.not_equal(
total_per_class_weighted_ious +
total_per_class_gt_areas, 0))
op = tf.summary.scalar(
metric_name,
tf.reduce_mean(
tf.boolean_mask(per_class_covering, valid_classes)))
op = tf.Print(op, [per_class_covering],
metric_name + '_classwise: ',
summarize=dataset.num_of_classes)
op = tf.Print(op, [
tf.reduce_mean(
tf.boolean_mask(per_class_covering, valid_classes))
],
metric_name + '_mean: ',
summarize=1)
summary_ops.append(op)
else:
raise ValueError('The metric_name "%s" is not supported.' %
metric_name)
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
metric_values = slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=FLAGS.checkpoint_dir,
logdir=FLAGS.eval_logdir,
num_evals=20,
eval_op=metrics_to_updates.values(),
final_op=metrics_to_values.values(),
summary_op=tf.summary.merge(summary_ops),
max_number_of_evaluations=FLAGS.max_number_of_evaluations,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset( # 获取验证集图片数据
dataset_name=FLAGS.dataset, # 数据集名称 cityscapes 默认为 pascal_voc_seg
split_name=FLAGS.eval_split, # 指定带有val的tfrecorder数据集 默认为“val”
dataset_dir=FLAGS.dataset_dir, # 数据集目录 tfrecoder文件的数据集目录
batch_size=FLAGS.eval_batch_size, # 每个batch包含的image数量 默认为1
crop_size=[int(sz)
for sz in FLAGS.eval_crop_size], # 评估时crop_size 默认为513,513
min_resize_value=FLAGS.min_resize_value, # 默认为None
max_resize_value=FLAGS.max_resize_value, # 默认为None
resize_factor=FLAGS.resize_factor, # 默认为None
model_variant=FLAGS.model_variant, # 模型的变体 本次训练为 xception_65
num_readers=2, # 并行读取图片的数量
is_training=False, # 不训练
should_shuffle=False, # 不将输入的数据随机打乱
should_repeat=False) # 不一直重复
tf.gfile.MakeDirs(FLAGS.eval_logdir) # 创建评估目录
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next() # 获取一次迭代的验证集数据
'''
samples:
{'image_name': <tf.Tensor 'IteratorGetNext:2' shape=(?,) dtype=string>,
'width': <tf.Tensor 'IteratorGetNext:5' shape=(?,) dtype=int64>,
'image': <tf.Tensor 'IteratorGetNext:1' shape=(?, 1024, 2048, 3) dtype=float32>,
'height': <tf.Tensor 'IteratorGetNext:0' shape=(?,) dtype=int64>,
'label': <tf.Tensor 'IteratorGetNext:3' shape=(?, 1024, 2048, 1) dtype=int32>,
'original_image': <tf.Tensor 'IteratorGetNext:4' shape=(?, ?, ?, 3) dtype=uint8>}
'''
model_options = common.ModelOptions( # 模型参数
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
}, # {semantic: 19}
crop_size=[int(sz) for sz in FLAGS.eval_crop_size], # 1024,2048
atrous_rates=FLAGS.atrous_rates, # 6,12,18
output_stride=FLAGS.output_stride) # 16
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape( # 设置形状
[
FLAGS.eval_batch_size, # 默认为1
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]),
3
])
if tuple(FLAGS.eval_scales) == (1.0, ): # 默认 评估尺度为1
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options, # 进行每个像素点预测
image_pyramid=FLAGS.image_pyramid)
'''
predictions:
{'semantic': <tf.Tensor 'ArgMax:0' shape=(1, 1024, 2048) dtype=int64>,
'semantic_prob': <tf.Tensor 'Softmax:0' shape=(1, 1024, 2048, 19) dtype=float32>}
'''
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1]) # 预测标签
labels = tf.reshape(samples[common.LABEL], shape=[-1]) # 真实标签
weights = tf.to_float(tf.not_equal(labels,
dataset.ignore_label)) # 各标签权重
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou' # MIoU
predictions_tag1 = 'accuracy_pixel' # 像素精度
for eval_scale in FLAGS.eval_scales: # 默认为单尺度[1.0]
predictions_tag += '_' + str(eval_scale)
predictions_tag1 += '_' + str(eval_scale)
if FLAGS.add_flipped_images: # 默认为False 不设置左右翻转来评估模型
predictions_tag += '_flipped'
predictions_tag1 += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes # 19
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
'''
metric_map:
{'eval/miou_1.0_overall': (<tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
<tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>)}
'''
metric_map['eval/%s_overall_accuracy_' %
predictions_tag] = tf.metrics.accuracy(
labels=labels, predictions=predictions, weights=weights)
# IoU for each class.
'''
tf.one_hot(indices, depth, on_value=None, off_value=None, axis=None, dtype=None, name=None)
Returns a one-hot tensor.
ndices表示输入的多个数值，通常是矩阵形式；depth表示输出的尺寸。
'''
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes]) # 预测输出的one_hot
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels,
[-1, num_classes]) # 真实label的one_hot
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c
) # miou_1.0_class_c
predictions_tag_c1 = '%s_class_%d' % (predictions_tag1, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tn, tn_op = tf.metrics.true_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
ap = tf.where(tf.greater(tp + fn, 0.0),
(tp + tn) / (tp + tn + fn + fp), tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
metric_map['eval/%s' % predictions_tag_c1] = (ap, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
'''
(metrics_to_values, metrics_to_updates):
({'eval/miou_1.0_class_5': <tf.Tensor 'Select_6:0' shape=() dtype=float32>,
'eval/miou_1.0_class_18': <tf.Tensor 'Select_19:0' shape=() dtype=float32>,
'eval/miou_1.0_class_13': <tf.Tensor 'Select_14:0' shape=() dtype=float32>,
'eval/miou_1.0_class_1': <tf.Tensor 'Select_2:0' shape=() dtype=float32>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_17': <tf.Tensor 'Select_18:0' shape=() dtype=float32>,
'eval/miou_1.0_class_8': <tf.Tensor 'Select_9:0' shape=() dtype=float32>,
'eval/miou_1.0_class_2': <tf.Tensor 'Select_3:0' shape=() dtype=float32>,
'eval/miou_1.0_class_0': <tf.Tensor 'Select_1:0' shape=() dtype=float32>,
'eval/miou_1.0_class_3': <tf.Tensor 'Select_4:0' shape=() dtype=float32>,
'eval/miou_1.0_class_14': <tf.Tensor 'Select_15:0' shape=() dtype=float32>,
'eval/miou_1.0_class_11': <tf.Tensor 'Select_12:0' shape=() dtype=float32>,
'eval/miou_1.0_class_6': <tf.Tensor 'Select_7:0' shape=() dtype=float32>,
'eval/miou_1.0_class_15': <tf.Tensor 'Select_16:0' shape=() dtype=float32>,
'eval/miou_1.0_class_4': <tf.Tensor 'Select_5:0' shape=() dtype=float32>,
'eval/miou_1.0_class_9': <tf.Tensor 'Select_10:0' shape=() dtype=float32>,
'eval/miou_1.0_class_16': <tf.Tensor 'Select_17:0' shape=() dtype=float32>,
'eval/miou_1.0_class_7': <tf.Tensor 'Select_8:0' shape=() dtype=float32>,
'eval/miou_1.0_class_10': <tf.Tensor 'Select_11:0' shape=() dtype=float32>,
'eval/miou_1.0_class_12': <tf.Tensor 'Select_13:0' shape=() dtype=float32>},
{'eval/miou_1.0_class_5': <tf.Operation 'group_deps_5' type=NoOp>,
'eval/miou_1.0_class_18': <tf.Operation 'group_deps_18' type=NoOp>,
'eval/miou_1.0_class_13': <tf.Operation 'group_deps_13' type=NoOp>,
'eval/miou_1.0_class_1': <tf.Operation 'group_deps_1' type=NoOp>,
'eval/miou_1.0_overall': <tf.Tensor 'mean_iou/AssignAdd:0' shape=(19, 19) dtype=float64_ref>,
'eval/miou_1.0_class_17': <tf.Operation 'group_deps_17' type=NoOp>,
'eval/miou_1.0_class_8': <tf.Operation 'group_deps_8' type=NoOp>,
'eval/miou_1.0_class_2': <tf.Operation 'group_deps_2' type=NoOp>,
'eval/miou_1.0_class_0': <tf.Operation 'group_deps' type=NoOp>,
'eval/miou_1.0_class_3': <tf.Operation 'group_deps_3' type=NoOp>,
'eval/miou_1.0_class_14': <tf.Operation 'group_deps_14' type=NoOp>,
'eval/miou_1.0_class_11': <tf.Operation 'group_deps_11' type=NoOp>,
'eval/miou_1.0_class_6': <tf.Operation 'group_deps_6' type=NoOp>,
'eval/miou_1.0_class_15': <tf.Operation 'group_deps_15' type=NoOp>,
'eval/miou_1.0_class_4': <tf.Operation 'group_deps_4' type=NoOp>,
'eval/miou_1.0_class_9': <tf.Operation 'group_deps_9' type=NoOp>,
'eval/miou_1.0_class_16': <tf.Operation 'group_deps_16' type=NoOp>,
'eval/miou_1.0_class_7': <tf.Operation 'group_deps_7' type=NoOp>,
'eval/miou_1.0_class_10': <tf.Operation 'group_deps_10' type=NoOp>,
'eval/miou_1.0_class_12': <tf.Operation 'group_deps_12' type=NoOp>})
'''
'''
tf.Print(input, data, message=None, first_n=None, summarize=None, name=None)
最低要求两个输入，input和data，input是需要打印的变量的名字，data要求是一个list，里面包含要打印的内容。
'''
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value) # 显示标量信息
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0: # 为0 暂不考虑
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0: # -1 暂不考虑
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
tf.logging.info('Prepare to export model to: %s', FLAGS.export_path)
with tf.Graph().as_default():
image, image_size, resized_image_size = _create_input_tensors()
model_options = common.ModelOptions(
outputs_to_num_classes={common.OUTPUT_TYPE: FLAGS.num_classes},
crop_size=FLAGS.crop_size,
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
if tuple(FLAGS.inference_scales) == (1.0, ):
tf.logging.info('Exported model performs single-scale inference.')
predictions = model.predict_labels(
image,
model_options=model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Exported model performs multi-scale inference.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
image,
model_options=model_options,
eval_scales=FLAGS.inference_scales,
add_flipped_images=FLAGS.add_flipped_images)
raw_predictions = tf.identity(
tf.cast(predictions[common.OUTPUT_TYPE], tf.float32),
_RAW_OUTPUT_NAME)
raw_probabilities = tf.identity(
predictions[common.OUTPUT_TYPE + model.PROB_SUFFIX],
_RAW_OUTPUT_PROB_NAME)
# Crop the valid regions from the predictions.
semantic_predictions = raw_predictions[:, :resized_image_size[0], :
resized_image_size[1]]
semantic_probabilities = raw_probabilities[:, :resized_image_size[0], :
resized_image_size[1]]
# Resize back the prediction to the original image size.
def _resize_label(label, label_size):
# Expand dimension of label to [1, height, width, 1] for resize operation.
label = tf.expand_dims(label, 3)
resized_label = tf.image.resize_images(
label,
label_size,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
align_corners=True)
return tf.cast(tf.squeeze(resized_label, 3), tf.int32)
semantic_predictions = _resize_label(semantic_predictions, image_size)
semantic_predictions = tf.identity(semantic_predictions,
name=_OUTPUT_NAME)
semantic_probabilities = tf.image.resize_bilinear(
semantic_probabilities,
image_size,
align_corners=True,
name=_OUTPUT_PROB_NAME)
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
saver = tf.train.Saver(tf.all_variables())
dirname = os.path.dirname(FLAGS.export_path)
tf.gfile.MakeDirs(dirname)
graph_def = tf.get_default_graph().as_graph_def(add_shapes=True)
freeze_graph.freeze_graph_with_def_protos(
graph_def,
saver.as_saver_def(),
FLAGS.checkpoint_path,
_OUTPUT_NAME + ',' + _OUTPUT_PROB_NAME,
restore_op_name=None,
filename_tensor_name=None,
output_graph=FLAGS.export_path,
clear_devices=True,
initializer_nodes=None)
if FLAGS.save_inference_graph:
tf.train.write_graph(graph_def, dirname, 'inference_graph.pbtxt')
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False,
with_cls=True,
cls_only=False,
output_valid=True)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
raise NotImplementedError('Multi-scale is not supported yet!')
metric_map = {}
## Extract cls logits
if FLAGS.weakly:
_, end_points = feature_extractor.extract_features(
samples[common.IMAGE],
output_stride=model_options.output_stride,
multi_grid=model_options.multi_grid,
model_variant=model_options.model_variant,
depth_multiplier=model_options.depth_multiplier,
divisible_by=model_options.divisible_by,
reuse=tf.AUTO_REUSE,
is_training=False,
preprocessed_images_dtype=model_options.
preprocessed_images_dtype,
global_pool=True,
num_classes=dataset.num_of_classes - 1)
# ResNet beta version has an additional suffix in FLAGS.model_variant, but
# it shares the same variable names with original version. Add a special
# handling here for beta version ResNet.
logits = end_points['{}/logits'.format(
FLAGS.model_variant).replace('_beta', '')]
logits = tf.reshape(logits, [-1, dataset.num_of_classes - 1])
cls_pred = tf.sigmoid(logits)
# Multi-label classification evaluation
cls_label = samples['cls_label']
cls_pred = tf.cast(tf.greater_equal(cls_pred, 0.5), tf.int32)
## For classification
metric_map['eval/cls_overall'] = tf.metrics.accuracy(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_precision'] = tf.metrics.precision(
labels=cls_label, predictions=cls_pred)
metric_map['eval/cls_recall'] = tf.metrics.recall(
labels=cls_label, predictions=cls_pred)
## For segmentation branch eval
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
# Define the evaluation metric.
num_classes = dataset.num_of_classes
## For segmentation
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
model_name=FLAGS.model_name,
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
for eval_scale in FLAGS.eval_scales:
predictions_tag += '_' + str(eval_scale)
if FLAGS.add_flipped_images:
predictions_tag += '_flipped'
# Define the evaluation metric.
metric_map = {}
num_classes = dataset.num_of_classes
metric_map['eval/%s_overall' % predictions_tag] = tf.metrics.mean_iou(
labels=labels,
predictions=predictions,
num_classes=num_classes,
weights=weights)
# IoU for each class.
one_hot_predictions = tf.one_hot(predictions, num_classes)
one_hot_predictions = tf.reshape(one_hot_predictions,
[-1, num_classes])
one_hot_labels = tf.one_hot(labels, num_classes)
one_hot_labels = tf.reshape(one_hot_labels, [-1, num_classes])
for c in range(num_classes):
predictions_tag_c = '%s_class_%d' % (predictions_tag, c)
tp, tp_op = tf.metrics.true_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fp, fp_op = tf.metrics.false_positives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
fn, fn_op = tf.metrics.false_negatives(
labels=one_hot_labels[:, c],
predictions=one_hot_predictions[:, c],
weights=weights)
tp_fp_fn_op = tf.group(tp_op, fp_op, fn_op)
iou = tf.where(tf.greater(tp + fn, 0.0), tp / (tp + fn + fp),
tf.constant(np.NaN))
metric_map['eval/%s' % predictions_tag_c] = (iou, tp_fp_fn_op)
(metrics_to_values,
metrics_to_updates) = contrib_metrics.aggregate_metric_map(metric_map)
summary_ops = []
for metric_name, metric_value in six.iteritems(metrics_to_values):
op = tf.summary.scalar(metric_name, metric_value)
op = tf.Print(op, [metric_value], metric_name)
summary_ops.append(op)
summary_op = tf.summary.merge(summary_ops)
summary_hook = contrib_training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
contrib_quantize.create_eval_graph()
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
contrib_tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=contrib_tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
contrib_training.evaluate_repeatedly(
checkpoint_dir=FLAGS.checkpoint_dir,
master=FLAGS.master,
eval_ops=list(metrics_to_updates.values()),
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def main(_):
if not FLAGS.model_name:
raise ValueError("Please specify a --model_name")
elif not FLAGS.checkpoint_path:
raise ValueError("Please specify a --checkpoint_path")
elif not FLAGS.eval_dir:
raise ValueError("Please specify an --eval_dir")
else:
pass
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
# Specify which TFRecord components to parse
keys_to_features = {
"image/encoded":
tf.FixedLenFeature((), tf.string, default_value=""),
"image/format":
tf.FixedLenFeature((), tf.string, default_value="png"),
"image/class/label":
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
"image": slim.tfexample_decoder.Image(),
"label": slim.tfexample_decoder.Tensor("image/class/label"),
}
items_to_descs = {
"image": "Color image",
"label": "Class idx",
}
label_idx_to_name = {}
for i, label in enumerate(CLASSES):
label_idx_to_name[i] = label
decoder = slim.tfexample_decoder.TFExampleDecoder(
keys_to_features, items_to_handlers)
file_pattern = "tfm_clf_%s.*"
file_pattern = os.path.join(FLAGS.dataset_dir,
file_pattern % FLAGS.dataset_split_name)
dataset = slim.dataset.Dataset(
data_sources=file_pattern,
reader=tf.TFRecordReader,
decoder=decoder,
num_samples=50,
items_to_descriptions=items_to_descs,
num_classes=len(CLASSES),
labels_to_names=label_idx_to_name,
)
# Create model
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=dataset.num_classes,
is_training=False)
# Create a dataset provider to load data from the dataset
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size,
)
[image, label] = provider.get(["image", "label"])
# Select the preprocessing function
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name,
is_training=False,
use_grayscale=FLAGS.use_grayscale)
eval_image_size = FLAGS.eval_image_size or network_fn.default_image_size
image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size,
)
# Define the model
logits, _ = network_fn(images)
if FLAGS.quantize:
contrib_quantize.create_eval_graph()
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
predictions = tf.argmax(logits, 1)
labels = tf.squeeze(labels)
# Define the metrics:
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
"Accuracy":
slim.metrics.streaming_accuracy(predictions, labels),
"Recall_5":
slim.metrics.streaming_recall_at_k(logits, labels, 5),
})
# Print the summaries to screen.
summary_ops = []
for name, value in names_to_values.items():
summary_name = "eval/%s" % name
op = tf.summary.scalar(summary_name, value, collections=[])
op = tf.Print(op, [value], summary_name)
summary_ops.append(op)
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
# This ensures that we make a single pass over all of the data.
num_batches = math.ceil(dataset.num_samples /
float(FLAGS.batch_size))
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info("Evaluating %s" % checkpoint_path)
slim.evaluation.evaluation_loop(
FLAGS.master,
FLAGS.checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
summary_op=tf.summary.merge(summary_ops),
eval_interval_secs=30)
