def fbeta(labels, predictions, num_classes, pos_indices=None, weights=None,
average='micro', beta=1):
"""Multi-class fbeta metric for Tensorflow
Parameters
----------
labels : Tensor of tf.int32 or tf.int64
The true labels
predictions : Tensor of tf.int32 or tf.int64
The predictions, same shape as labels
num_classes : int
The number of classes
pos_indices : list of int, optional
The indices of the positive classes, default is all
weights : Tensor of tf.int32, optional
Mask, must be of compatible shape with labels
average : str, optional
'micro': counts the total number of true positives, false
positives, and false negatives for the classes in
`pos_indices` and infer the metric from it.
'macro': will compute the metric separately for each class in
`pos_indices` and average. Will not account for class
imbalance.
'weighted': will compute the metric separately for each class in
`pos_indices` and perform a weighted average by the total
number of true labels for each class.
beta : int, optional
Weight of precision in harmonic mean
Returns
-------
tuple of (scalar float Tensor, update_op)
"""
cm, op = _streaming_confusion_matrix(
labels, predictions, num_classes, weights)
_, _, fbeta = metrics_from_confusion_matrix(
cm, pos_indices, average=average, beta=beta)
_, _, op = metrics_from_confusion_matrix(
op, pos_indices, average=average, beta=beta)
return (fbeta, op)
def precision(labels, predictions, num_classes, pos_indices=None,
weights=None, average='micro'):
"""Multi-class precision metric for Tensorflow
Parameters
----------
labels : Tensor of tf.int32 or tf.int64
The true labels
predictions : Tensor of tf.int32 or tf.int64
The predictions, same shape as labels
num_classes : int
The number of classes
pos_indices : list of int, optional
The indices of the positive classes, default is all
weights : Tensor of tf.int32, optional
Mask, must be of compatible shape with labels
average : str, optional
'micro': counts the total number of true positives, false
positives, and false negatives for the classes in
`pos_indices` and infer the metric from it.
'macro': will compute the metric separately for each class in
`pos_indices` and average. Will not account for class
imbalance.
'weighted': will compute the metric separately for each class in
`pos_indices` and perform a weighted average by the total
number of true labels for each class.
Returns
-------
tuple of (scalar float Tensor, update_op)
"""
cm, op = _streaming_confusion_matrix(
labels, predictions, num_classes, weights)
pr, _, _ = metrics_from_confusion_matrix(
cm, pos_indices, average=average)
op, _, _ = metrics_from_confusion_matrix(
op, pos_indices, average=average)
return (pr, op)
def fbeta(labels,
predictions,
num_classes,
pos_indices=None,
weights=None,
average='micro',
beta=1):
"""
pos_indices : list of int, optional
The indices of the positive classes, default is all
weights : Tensor of tf.int32, optional
Mask, must be of compatible shape with labels
average : str, optional
'micro': counts the total number of true positives, false
positives, and false negatives for the classes in
`pos_indices` and infer the metric from it.
'macro': will compute the metric separately for each class in
`pos_indices` and average. Will not account for class
imbalance.
'weighted': will compute the metric separately for each class in
`pos_indices` and perform a weighted average by the total
number of true labels for each class.
beta : int, optional
Weight of precision in harmonic mean
"""
cm, op = _streaming_confusion_matrix(labels, predictions, num_classes,
weights)
_, _, fbeta = metrics_from_confusion_matrix(cm,
pos_indices,
average=average,
beta=beta)
_, _, op = metrics_from_confusion_matrix(op,
pos_indices,
average=average,
beta=beta)
return (fbeta, op)
def define_estimator(mode, features, labels, model_fn, config, params):
"""Add documentation... More information at tf.Estimator class.
Assumptions:
features: a dict containing rawfeatures and profeatures
both: Nb x hf x wf x 3, tf.float32, in [0,1]
labels: a dict containing rawfeatures and profeatures
both: Nb x hf x wf, tf.int32, in [0,Nc-1]
Args:
features: First item returned by input_fn passed to train, evaluate, and predict.
labels: Second item returned by input_fn passed to train, evaluate, and predict.
mode: one of tf.estimator.ModeKeys.
config: a tf.estimator.RunConfig object...
parameters: a tf.train.HParams object...
...
"""
assert mode in _ALLOWED_MODES, (
'mode should be TRAIN, EVAL or PREDICT from tf.estimator.ModeKeys.')
assert params.name_feature_extractor in {
'resnet_v1_50', 'resnet_v1_101'
}, ('params must have name_feature_extractor attribute in resnet_v1_{50,101}.'
)
if params.name_feature_extractor == 'resnet_v1_101':
raise NotImplementedError(
'Use of resnet_v1_101 as base feature extractor is not yet implemented.'
)
# unpack features
rawimages = features['rawimages'] if 'rawimages' in features.keys(
) else None
rawimagespaths = features[
'rawimagespaths'] if 'rawimagespaths' in features.keys() else None
proimages = features['proimages']
prolabels = labels if labels else None
## build a fully convolutional model for semantic segmentation
# predictions refer to the training class ids
# for plotting of results (inference) or assessment, predictions should be transformed
# using `{inference, evaluation}_problem_def`s
_, _, predictions = model_fn(mode, proimages, prolabels, config, params)
# TODO(panos): assert that proimages and predictions have same spatial size
if mode == tf.estimator.ModeKeys.TRAIN:
# global step
global_step = tf.train.get_or_create_global_step()
# losses
with tf.variable_scope('losses'):
losses = define_losses(mode, predictions, prolabels, config,
params)
# exponential moving averages
# creates variables in checkpoint with name: 'emas/' + <variable_name> +
# {'ExponentialMovingAverage,Momentum}
# ex.: for 'classifier/logits/Conv/biases' it saves also
# 'emas/classifier/logits/Conv/biases/ExponentialMovingAverage'
# and 'emas/classifier/logits/Conv/biases/Momentum'
# create_train_op guarantees to run GraphKeys.UPDATE_OPS collection
# before total_loss in every step, but doesn't give any guarantee
# for running after some other op, and since ema need to be run
# after applying the gradients maybe this code needs checking
if params.ema_decay > 0:
with tf.variable_scope('exponential_moving_averages'):
#for mv in slim.get_model_variables():
# print('slim.model_vars:', mv.op.name)
ema = tf.train.ExponentialMovingAverage(
params.ema_decay,
num_updates=global_step,
zero_debias=True)
variables_to_ema = []
for mv in tf.model_variables():
if 'BatchNorm/moving' not in mv.name:
variables_to_ema.append(mv)
print(
f"\nFound {len(tf.model_variables())} variables, saving exponential "
f"moving averages for {len(variables_to_ema)} of them.\n")
maintain_ema_op = ema.apply(var_list=variables_to_ema)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, maintain_ema_op)
# create training operation
with tf.variable_scope('train_ops'):
# optimizer
optimizer = define_optimizer(global_step, params)
# training op
train_op = create_train_op(
losses['total'],
optimizer,
global_step=global_step,
# update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS),
summarize_gradients=False,
# transform_grads_fn=,
# gradient_multipliers=gradient_multipliers,
check_numerics=False,
)
# TODO: maybe parameterize it
training_hooks = [
_RunMetadataHook(params.log_dir,
every_n_iter=max(params.num_training_steps // 50,
params.save_checkpoints_steps))
]
# next two lines were added for distributed debugging
if params.distribute:
tower_context = tf.contrib.distribute.get_tower_context()
assert tower_context
print(
f"Tower {tower_context.tower_id}: _RunMetadataHook is not supported "
"yet for distributed training.")
training_hooks = []
replace_initializers(config, params)
summaries_data = {
'features': features,
'labels': labels,
'predictions': predictions,
'losses': losses,
'learning_rate': optimizer._learning_rate
} #pylint: disable=protected-access
scaffold = _define_scaffold(mode, config, params, summaries_data)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
train_op=train_op,
training_hooks=training_hooks,
scaffold=scaffold)
if mode == tf.estimator.ModeKeys.EVAL:
with tf.variable_scope('losses'):
losses = define_losses(mode, predictions, prolabels, config,
params)
# returns (variable, update_op)
# TF internal error/problem: _streaming_confusion_matrix internally casts
# labels and predictions to int64, and since we feed a dictionary, tensors are
# passed by reference leading them to change type, thus we send an identity
# confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
# tf.identity(prolabels),
# tf.identity(predictions['decisions']),
# params.output_Nclasses)
# l1_probs, decs = itemgetter('l1_probabilities', 'decisions')(predictions)
# create a new dict with the supported keys only
predictions = _map_predictions_to_new_cids(
predictions, params.training_cids2evaluation_cids)
if params.replace_voids:
predictions = _replace_voids(predictions, params)
# TODO(panos): confusion matrix expects prolabels and predictions to have the same shape
# this may not the case when preserve_aspect_ratio is set and this will give an error
if hasattr(params, 'preserve_aspect_ratio'):
if params.preserve_aspect_ratio:
raise NotImplementedError(
'evaluation with preserving aspect ratio is not implemented.'
)
predictions = _resize_predictions(predictions,
tf.shape(labels['prolabels'])[1:3],
params)
tcids2ecids = _replacevoids(params.training_cids2evaluation_cids)
confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
labels['prolabels'],
predictions['decisions'],
# +1 due to convention of starting counting at 0
max(tcids2ecids) + 1)
# dict of metrics keyed by name with values tuples of (metric_tensor, update_op)
# TODO: add more semantic segmentation metrics
eval_metric_ops = {
'confusion_matrix':
(tf.to_int32(confusion_matrix[0]), confusion_matrix[1])
}
scaffold = _define_scaffold(mode, config, params)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
eval_metric_ops=eval_metric_ops,
scaffold=scaffold)
if mode == tf.estimator.ModeKeys.PREDICT:
# create a new dict with the supported keys only
l1_probs, l2_vehicle_probs, l2_human_probs, decs = itemgetter(
'l1_probabilities', 'l2_vehicle_probabilities',
'l2_human_probabilities', 'decisions')(predictions)
predictions = {
'l1_probabilities': l1_probs,
'l2_vehicle_probabilities': l2_vehicle_probs,
'l2_human_probabilities': l2_human_probs,
'decisions': decs
}
# workaround for connecting input pipeline outputs to system output
# TODO(panos): maybe from a system perspective makes more sense to have mapping and
# resizing in the system_factory
# since these are functions of the system and not the network/estimator
# new size defaults to provided values
# if at least one is None then new size is the arbitrary size of rawimage in each step
new_size = (params.height_system, params.width_system)
is_arbitrary = not all(new_size)
if is_arbitrary:
if rawimages is not None:
predictions['rawimages'] = rawimages
if rawimagespaths is not None:
predictions['rawimagespaths'] = rawimagespaths
new_size = tf.shape(predictions['rawimages'])[1:3]
predictions = _resize_predictions(predictions, new_size, params)
tf.logging.warn(
'Mapping of predictions to new cids is not implemented for now.')
# predictions = _map_predictions_to_new_cids(predictions, params.training_cids2inference_cids)
if params.replace_voids:
predictions = _replace_voids(predictions, params)
scaffold = _define_scaffold(mode, config, params)
estimator_spec = tf.estimator.EstimatorSpec(mode,
predictions=predictions,
scaffold=scaffold)
return estimator_spec
def define_estimator(mode, features, labels, model_fn, config, params):
"""Add documentation... More information at tf.Estimator class.
Assumptions:
features: a dict containing rawfeatures and profeatures
both: Nb x hf x wf x 3, tf.float32, in [0,1]
labels: a dict containing rawfeatures and profeatures
both: Nb x hf x wf, tf.int32, in [0,Nc-1]
Args:
features: First item returned by input_fn passed to train, evaluate, and predict.
labels: Second item returned by input_fn passed to train, evaluate, and predict.
mode: one of tf.estimator.ModeKeys.
model_fn: the model function that maps images to predictions
config: a tf.estimator.RunConfig object...
params: a tf.train.HParams object...
...
"""
assert mode in _ALLOWED_MODES, (
'mode should be TRAIN, EVAL or PREDICT from tf.estimator.ModeKeys.')
assert params.name_feature_extractor in {'resnet_v1_50', 'resnet_v1_101'}, (
'params must have name_feature_extractor attribute in resnet_v1_{50,101}.')
# unpack features
proimages = features['proimages']
prolabels = labels['prolabels'] if mode != tf.estimator.ModeKeys.PREDICT else None
# -- build a fully convolutional model for semantic segmentation
_, _, predictions = model_fn(mode, proimages, prolabels, config, params)
# create training ops and exponential moving averages
if mode == tf.estimator.ModeKeys.TRAIN:
if params.switch_train_op:
with tf.variable_scope('domain_labels'):
# Manually create the domain labels
# TODO (rob) work around these domain labels with custom domain loss ??
num_feature_vecs = int(params.height_feature_extractor *
params.width_feature_extractor / params.stride_feature_extractor ** 2)
domain_labels_tiled = tf.tile(tf.expand_dims(labels['domainlabels'], 1), [1, num_feature_vecs])
domain_labels = tf.reshape(domain_labels_tiled, [-1])
labels['domainlabels'] = domain_labels
else:
domain_labels = None
# global step
global_step = tf.train.get_or_create_global_step()
# losses
with tf.variable_scope('losses'):
losses = define_losses(mode, config, params, predictions, prolabels, domain_labels)
# exponential moving averages
# creates variables in checkpoint with name: 'emas/' + <variable_name> +
# {'ExponentialMovingAverage,Momentum}
# ex.: for 'classifier/logits/Conv/biases' it saves also
# 'emas/classifier/logits/Conv/biases/ExponentialMovingAverage'
# and 'emas/classifier/logits/Conv/biases/Momentum'
# create_train_op guarantees to run GraphKeys.UPDATE_OPS collection
# before total_loss in every step, but doesn't give any guarantee
# for running after some other op, and since ema need to be run
# after applying the gradients maybe this code needs checking
if params.ema_decay > 0:
with tf.name_scope('exponential_moving_averages'):
# for mv in slim.get_model_variables():
# log.debug('slim.model_vars:', mv.op.name)
log.debug('Record exponential weighted moving averages')
ema = tf.train.ExponentialMovingAverage(params.ema_decay,
num_updates=global_step,
zero_debias=True)
maintain_ema_op = ema.apply(var_list=list(filter(lambda x: 'domain_classifier' not in x.name,
tf.trainable_variables())))
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, maintain_ema_op)
# create training operation
with tf.variable_scope('train_ops'):
learning_rate = tf.train.piecewise_constant(global_step,
params.lr_boundaries,
params.lr_values)
# optimizer
if params.optimizer == 'SGDM':
optimizer = tf.train.MomentumOptimizer(
learning_rate,
params.momentum,
use_nesterov=params.use_nesterov)
elif params.optimizer == 'SGD':
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
else:
log.warning('Optimizer type not found (%s)' % params.optimizer)
optimizer = None
# training op
train_op = create_alternating_train_op(losses, optimizer, global_step, params)
training_hooks = []
# _RunMetadataHook(params.log_dir,
# every_n_iter=max(params.num_training_steps // 50,
# params.save_checkpoints_steps))]
summaries_data = {'features': features,
'labels': labels,
'predictions': predictions,
'losses': losses,
'learning_rate': learning_rate}
if mode == tf.estimator.ModeKeys.EVAL:
with tf.variable_scope('losses'):
losses = define_losses(mode, config, params, predictions, prolabels)
# returns (variable, update_op)
# TF internal error/problem: _streaming_confusion_matrix internally casts
# labels and predictions to int64, and since we feed a dictionary, tensors are
# passed by reference leading them to change type, thus we send an identity
# confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
# tf.identity(prolabels),
# tf.identity(predictions['decisions']),
# params.training_Nclasses)
confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
prolabels,
predictions['decisions'],
params.training_Nclasses)
# dict of metrics keyed by name with values tuples of (metric_tensor, update_op)
eval_metric_ops = {'confusion_matrix': (
tf.to_int32(confusion_matrix[0]), confusion_matrix[1])}
# -- create EstimatorSpec according to mode
if mode == tf.estimator.ModeKeys.TRAIN:
scaffold = _define_scaffold(mode, config, params, summaries_data)
# training_hooks.append(CheckpointSaverHookCustom(start_step=int(params.num_batches_per_epoch * (params.Ne - 1)),
# checkpoint_dir=params.log_dir,
# scaffold=scaffold,
# save_steps=int(params.num_batches_per_epoch / 5)))
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
train_op=train_op,
training_hooks=training_hooks,
scaffold=scaffold)
elif mode == tf.estimator.ModeKeys.EVAL:
scaffold = _define_scaffold(mode, config, params)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
eval_metric_ops=eval_metric_ops,
scaffold=scaffold)
elif mode == tf.estimator.ModeKeys.PREDICT:
scaffold = _define_scaffold(mode, config, params)
# workaround for connecting input pipeline outputs to system output
# TODO: make it more clear
predictions['rawimages'] = features['rawimages']
predictions['rawimagespaths'] = features['rawimagespaths']
# the expected predictions.keys() in this point is:
# dict_keys(['logits', 'probabilities', 'decisions', 'rawimages', 'rawimagespaths'])
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
scaffold=scaffold)
else:
assert False, f'No such mode {mode}'
return estimator_spec
import numpy as np
import tensorflow as tf
from tensorflow.python.ops.metrics_impl import _streaming_confusion_matrix
def get_metrics_ops(labels, predictions, num_labels):
# 得到混淆矩阵和update_op，在这里我们需要将生成的混淆矩阵转换成tensor
cm, op = _streaming_confusion_matrix(labels, predictions, num_labels)
tf.logging.info(type(cm))
tf.logging.info(type(op))
return (tf.convert_to_tensor(cm), op)
def get_metrics(conf_mat, num_labels):
# 得到numpy类型的混淆矩阵，然后计算precision，recall，f1值。
precisions = []
recalls = []
for i in range(num_labels):
tp = conf_mat[i][i].sum()
col_sum = conf_mat[:, i].sum()
row_sum = conf_mat[i].sum()
precision = tp / col_sum if col_sum > 0 else 0
recall = tp / row_sum if row_sum > 0 else 0
precisions.append(precision)
recalls.append(recall)
pre = sum(precisions) / len(precisions)
rec = sum(recalls) / len(recalls)
def get_metrics_ops(labels, predictions, num_labels):
cm, op = _streaming_confusion_matrix(labels, predictions, num_labels)
tf.logging.info(type(cm))
tf.logging.info(type(op))
return (tf.convert_to_tensor(cm), op)
def mean_iou(labels,
predictions,
num_classes,
weights=None,
metrics_collections=None,
updates_collections=None,
name=None):
"""Calculate per-step mean Intersection-Over-Union (mIOU).
Mean Intersection-Over-Union is a common evaluation metric for
semantic image segmentation, which first computes the IOU for each
semantic class and then computes the average over classes.
IOU is defined as follows:
IOU = true_positive / (true_positive + false_positive + false_negative).
The predictions are accumulated in a confusion matrix,
weighted by `weights`, and mIOU is then calculated from it.
For estimation of the metric over a stream of data, the function
creates 7
an `update_op` operation that updates these variables and returns
the `mean_iou`. If `weights` is `None`, weights default to 1.
Use weights of 0 to mask values.
Args:
labels: A `Tensor` of ground truth labels with shape [batch size] and of
type `int32` or `int64`. The tensor will be flattened if its rank > 1.
predictions: A `Tensor` of prediction results for semantic labels, whose
shape is [batch size] and type `int32` or `int64`. The tensor will be
flattened if its rank > 1.
num_classes: The possible number of labels the prediction task can
have. This value must be provided, since a confusion matrix of
dimension = [num_classes, num_classes] will be allocated.
weights: Optional `Tensor` whose rank is either 0, or the same rank as
`labels`, and must be broadcastable to `labels` (i.e., all dimensions
must be either `1`, or the same as the corresponding `labels`
dimension).
metrics_collections: An optional list of collections that `mean_iou`
should be added to.
updates_collections: An optional list of collections `update_op` should
be added to.
name: An optional variable_scope name.
Returns:
mean_iou: A `Tensor` representing the mean intersection-over-union.
update_op: An operation that increments the confusion matrix.
Raises:
ValueError: If `predictions` and `labels` have mismatched shapes, or if
`weights` is not `None` and its shape doesn't match `predictions`,
or if either `metrics_collections` or `updates_collections` are not a
list or tuple.
"""
with variable_scope.variable_scope(
name, 'mean_iou', (predictions, labels, weights)):
# Check if shape is compatible.
predictions.get_shape().assert_is_compatible_with(labels.get_shape())
total_cm, update_op = _streaming_confusion_matrix(labels, predictions,
num_classes, weights)
reset_cm_op = tf.assign(total_cm, tf.zeros_like(total_cm,
total_cm.dtype,
'reset_cm'))
def compute_mean_iou(name):
"""Compute the mean intersection-over-union via the confusion
matrix."""
sum_over_row = math_ops.to_float(math_ops.reduce_sum(total_cm, 0))
sum_over_col = math_ops.to_float(math_ops.reduce_sum(total_cm, 1))
cm_diag = math_ops.to_float(array_ops.diag_part(total_cm))
denominator = sum_over_row + sum_over_col - cm_diag
# If the value of the denominator is 0, set it to 1 to avoid
# zero division.
denominator = array_ops.where(
math_ops.greater(denominator, 0),
denominator,
array_ops.ones_like(denominator))
iou = math_ops.div(cm_diag, denominator)
return math_ops.reduce_mean(iou, name=name), iou
mean_iou_v, iou = compute_mean_iou('mean_iou')
if metrics_collections:
ops.add_to_collections(metrics_collections, mean_iou_v)
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return mean_iou_v, iou, update_op, reset_cm_op
def define_estimator(mode, features, labels, model_fn, config, params):
"""
Assumptions:
features: a dict containing rawfeatures and profeatures
both: Nb x hf x wf x 3, tf.float32, in [0,1]
labels: a dict containing rawfeatures and profeatures
both: Nb x hf x wf, tf.int32, in [0,Nc-1]
Args:
features: First item returned by input_fn passed to train, evaluate, and predict.
labels: Second item returned by input_fn passed to train, evaluate, and predict.
mode: one of tf.estimator.ModeKeys.
"""
assert mode in _ALLOWED_MODES, (
'mode should be TRAIN, EVAL or PREDICT from tf.estimator.ModeKeys.')
assert params.name_feature_extractor in {
'resnet_v1_50', 'resnet_v1_101'
}, ('params must have name_feature_extractor attribute in resnet_v1_{50,101}.'
)
if params.name_feature_extractor == 'resnet_v1_101':
raise NotImplementedError(
'Use of resnet_v1_101 as base feature extractor is not yet implemented.'
)
# unpack features
rawimages = features['rawimages']
proimages = features['proimages']
# TODO: fix this temporary workaround for labels
# rawlabels = labels['rawlabels'] if mode != tf.estimator.ModeKeys.PREDICT else None
prolabels = labels[
'prolabels'] if mode != tf.estimator.ModeKeys.PREDICT else None
print('debug:rawimages:', rawimages)
print('debug:proimages:', proimages)
print('debug:prolabels:', prolabels)
## build a fully convolutional model for semantic segmentation
_, _, predictions = model_fn(mode, proimages, prolabels, config, params)
# print('debug: predictions:', predictions)
## create training ops and exponential moving averages
if mode == tf.estimator.ModeKeys.TRAIN:
# global step
global_step = tf.train.get_or_create_global_step()
# losses
with tf.variable_scope('losses'):
losses = define_losses(mode, config, params, predictions,
prolabels)
# exponential moving averages
# creates variables in checkpoint with name: 'emas/' + <variable_name> +
# {'ExponentialMovingAverage,Momentum}
# ex.: for 'classifier/logits/Conv/biases' it saves also
# 'emas/classifier/logits/Conv/biases/ExponentialMovingAverage'
# and 'emas/classifier/logits/Conv/biases/Momentum'
# create_train_op guarantees to run GraphKeys.UPDATE_OPS collection
# before total_loss in every step, but doesn't give any guarantee
# for running after some other op, and since ema need to be run
# after applying the gradients maybe this code needs checking
if params.ema_decay > 0:
with tf.name_scope('exponential_moving_averages'):
#for mv in slim.get_model_variables():
# print('slim.model_vars:', mv.op.name)
ema = tf.train.ExponentialMovingAverage(
params.ema_decay,
num_updates=global_step,
zero_debias=True)
maintain_ema_op = ema.apply(var_list=tf.model_variables())
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, maintain_ema_op)
# create training operation
with tf.variable_scope('train_ops'):
learning_rate = tf.train.piecewise_constant(
global_step, params.lr_boundaries, params.lr_values)
# optimizer
if params.optimizer == 'SGDM':
optimizer = tf.train.MomentumOptimizer(
learning_rate,
params.momentum,
use_nesterov=params.use_nesterov)
elif params.optimizer == 'SGD':
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
# training op
train_op = create_train_op(
losses['total'],
optimizer,
global_step=global_step,
#update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS),
# summarize_gradients=True,
# #clip_gradient_norm=params.clip_grad_norm,
# #gradient_multipliers=gradient_multipliers,
check_numerics=False,
)
# TODO: maybe parameterize it
training_hooks = [
_RunMetadataHook(params.log_dir,
every_n_iter=max(params.num_training_steps // 50,
params.save_checkpoints_steps))
]
summaries_data = {
'features': features,
'labels': labels,
'predictions': predictions,
'losses': losses,
'learning_rate': learning_rate
}
# flatten and concatenate decisions
if mode in [tf.estimator.ModeKeys.EVAL, tf.estimator.ModeKeys.PREDICT]:
# don't forget to change confusion matrix outputs
# C: 28, M: 66, G: 44, E: 71
flatten_decs = _flatten_all_decs(predictions['decisions'])
# flatten_decs = _flatten_for_cityscapes_val(predictions['decisions'])
# flatten_decs = _flatten_for_mapillary_val(predictions['decisions'])
# flatten_decs = _flatten_for_cityscapes_extended_val(predictions['decisions'])
# flatten_decs = _flatten_for_gtsdb_val(predictions['decisions'])
if mode == tf.estimator.ModeKeys.EVAL:
with tf.variable_scope('losses'):
losses = define_losses(mode, config, params, predictions,
prolabels)
# returns (variable, update_op)
# TF internal error/problem: _streaming_confusion_matrix internally casts
# labels and predictions to int64, and since we feed a dictionary, tensors are
# passed by reference leading them to change type, thus we send an identity
# confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
# tf.identity(prolabels),
# tf.identity(predictions['decisions']),
# params.training_Nclasses)
confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
prolabels, flatten_decs, 44)
# dict of metrics keyed by name with values tuples of (metric_tensor, update_op)
# TODO: add more semantic segmentation metrics
eval_metric_ops = {
'confusion_matrix':
(tf.to_int32(confusion_matrix[0]), confusion_matrix[1])
}
## create EstimatorSpec according to mode
# unpack predictions
if mode in [tf.estimator.ModeKeys.TRAIN, tf.estimator.ModeKeys.EVAL]:
predictions = None
else:
# redefine predictions according to estimator requirements
predictions = {
'logits': predictions['logits'][0][0],
'probabilities': predictions['probabilities'][0][0],
# 'decisions': predictions['decisions'][0],
'decisions': flatten_decs,
}
if mode == tf.estimator.ModeKeys.TRAIN:
scaffold = _define_scaffold(mode, config, params, summaries_data)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
train_op=train_op,
training_hooks=training_hooks,
scaffold=scaffold)
elif mode == tf.estimator.ModeKeys.EVAL:
scaffold = _define_scaffold(mode, config, params)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
eval_metric_ops=eval_metric_ops,
scaffold=scaffold)
elif mode == tf.estimator.ModeKeys.PREDICT:
scaffold = _define_scaffold(mode, config, params)
# workaround for connecting input pipeline outputs to system output
# TODO: make it more clear
predictions['rawimages'] = rawimages
predictions['rawimagespaths'] = features['rawimagespaths']
# the expected predictions.keys() in this point is:
# dict_keys(['logits', 'probabilities', 'decisions', 'rawimages', 'rawimagespaths'])
estimator_spec = tf.estimator.EstimatorSpec(mode,
predictions=predictions,
scaffold=scaffold)
return estimator_spec
def confusion_matrices_for_classes_and_subclasses(labels, probabilities):
"""VALID ONLY for 19 Cityscapes classes and 43 GTSDB subclasses assuming
63 trainIds: classIds: 0-18, signIds: 19-61, void: 62 (trafficSignId: 7).
Calculates the confusion matrix with special care in the traffic sign classes
and subclasses according to the scheme:
trainIds 0-18 (classes): all normal except 7 (traffic sign), if a pixel is TP
top1 labels (7 or 19-61) then it is counted as correct for class 7
trainIds 19-61 (subclasses): TP if top1 is in 19-61 or top1 is 7 and top2 in 19-61
labels: Nb x H x W, tf.int32, in [0,62]
probabilities: Nb x H x W x 62, tf.float32, in [0,1]
"""
probs = probabilities
# probs.get_shape().assert_is_compatible_with(labels[...,tf.newaxis].get_shape())
labels.get_shape().assert_has_rank(3)
probs.get_shape().assert_has_rank(4)
assert labels.dtype == tf.int32, f"labels dtype is {labels.dtype}"
assert probs.dtype == tf.float32, f"probs dtype is {probs.dtype}"
# if labels.dtype != probs.dtype:
# assert False, f"labels dtype ({labels.dtype}) doesn't match decisions ({decisions.dtype})"
#decisions = math_ops.cast(decisions, labels.dtype)
decs = tf.cast(tf.argmax(probs, 3), tf.int32, name='decisions')
# translate ground truth and decisions to classes
void_mask = tf.equal(labels, 62)
subclass_mask = tf.logical_and(labels >= 19, labels <= 61)
labels2classes = tf.where(
void_mask,
tf.ones_like(labels) * 19,
tf.where(subclass_mask,
tf.ones_like(labels) * 7, labels))
void_mask = tf.equal(decs, 62)
subclass_mask = tf.logical_and(decs >= 19, decs <= 61)
decs2classes = tf.where(
void_mask,
tf.ones_like(labels) * 19,
tf.where(subclass_mask,
tf.ones_like(decs) * 7, decs))
# if label is tsign (7, 19-61) and decision is tsign (7, 19-61) then correct
# for the rest classes keep as is
class_cm = metrics_impl._streaming_confusion_matrix(
labels2classes, decs2classes, 20)
# translate ground truth and decisions to subclasses
subclass_mask = tf.logical_and(labels >= 19, labels <= 61)
labels2subclasses = tf.where(subclass_mask, labels,
tf.ones_like(labels) * 62) - 19
subclass_mask = tf.logical_and(decs >= 19, decs <= 61)
tsign_class_mask = tf.equal(decs, 7)
_, i = tf.nn.top_k(probs, k=2) # 4D
# print('debug:i:', i.shape, i.dtype)
top2_is_subclass_mask = tf.logical_and(i[..., 1] >= 19,
i[..., 1] <= 61) # 3D
# print('debug:top2_and_any_subclass_mask:', top2_is_subclass_mask.shape, top2_is_subclass_mask.dtype)
# if top1 is subclass:
# keep top1 label
# else:
# if top1 is tsign class:
# if top2 is subclass:
# keep top2 label
# else:
# keep top1 label
# else:
# give void label (because we create labels for subclass evaluation only)
decs2subclasses = tf.where(
subclass_mask, decs,
tf.where(
tsign_class_mask,
tf.where(top2_is_subclass_mask, i[..., 1],
tf.ones_like(decs) * (7 + 19)),
tf.ones_like(decs) * 62)) - 19
subclass_cm = metrics_impl._streaming_confusion_matrix(
labels2subclasses, decs2subclasses, 44)
return class_cm, subclass_cm
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
tf.logging.info('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
tf.logging.info('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
tf.logging.info('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
for eval_scale in FLAGS.eval_scales:
predictions_tag += '_' + str(eval_scale)
if FLAGS.add_flipped_images:
predictions_tag += '_flipped'
# Define the evaluation metric.
miou, update_op = tf.metrics.mean_iou(predictions,
labels,
dataset.num_of_classes,
weights=weights)
tf.summary.scalar(predictions_tag, miou)
#new metric
from tensorflow.python.ops.metrics_impl import _streaming_confusion_matrix
cm, update_op_cm = _streaming_confusion_matrix(labels,
predictions,
dataset.num_of_classes,
weights=weights)
tf.summary.tensor_summary('confusion_matrix', cm)
#end new metric
summary_op = tf.summary.merge_all()
summary_hook = tf.contrib.training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
tf.contrib.quantize.create_eval_graph()
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
tf.contrib.training.evaluate_repeatedly(
master=FLAGS.master,
checkpoint_dir=FLAGS.checkpoint_dir,
eval_ops=[update_op, update_op_cm],
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
