def _train_prebatch_processing(rim, rla, params):
"""
TODO(panos): add more info...
Arguments:
rim: raw images, as extracted from tfrecords
rla: raw labels, as extracted from tfrecords
params: object with with the following attributes:
height_feature_extractor: ...
width_feature_extractor: ...
training_lids2cids: ...
"""
sfe = (params.height_feature_extractor, params.width_feature_extractor)
## prepare
rim = tf.image.convert_image_dtype(rim, dtype=tf.float32)
training_lids2cids = _replacevoids(params.training_problem_def['lids2cids'])
rla = tf.gather(tf.cast(training_lids2cids, tf.int32), tf.to_int32(rla))
## preprocess
rim.set_shape((None, None, None))
rla.set_shape((None, None))
proimages, prolabels = resize_images_and_labels(rim[tf.newaxis, ...],
rla[tf.newaxis, ...],
sfe,
preserve_aspect_ratio=params.preserve_aspect_ratio)
proimage, prolabel = proimages[0], prolabels[0]
# pre-batching augmentations
pass
return rim, rla, proimage, prolabel
def parse_args(self, argv):
# parse all arguments and add manually additional arguments
self.args = self._parser.parse_args(argv)
# problem definitions
self.args.training_problem_def = json.load(
open(self.args.training_problem_def_path, 'r'))
if hasattr(self.args, 'inference_problem_def_path'):
if self.args.inference_problem_def_path is None:
# TODO: implement the use of inference problem_def for training results in all functions
self.args.inference_problem_def = self.args.training_problem_def
else:
self.args.inference_problem_def = json.load(
open(self.args.inference_problem_def_path, 'r'))
if hasattr(self.args, 'evaluation_problem_def_path'):
if self.args.evaluation_problem_def_path is None:
self.args.evaluation_problem_def = self.args.training_problem_def
else:
self.args.evaluation_problem_def = json.load(
open(self.args.evaluation_problem_def_path, 'r'))
if hasattr(self.args, 'additional_problem_def_path'):
if self.args.additional_problem_def_path is None:
self.args.additional_problem_def = self.args.training_problem_def
else:
self.args.additional_problem_def = json.load(
open(self.args.additional_problem_def_path, 'r'))
# _validate_problem_config(self.args.config)
# by convention class ids start at 0
number_of_training_classes = max(
self.args.training_problem_def['lids2cids']) + 1
trained_with_void_class = -1 in self.args.training_problem_def[
'lids2cids']
self.args.training_Nclasses = number_of_training_classes + trained_with_void_class
self.args.training_lids2cids = _replacevoids(
self.args.training_problem_def['lids2cids'])
if hasattr(self.args, 'additional_problem_def'):
self.args.additional_lids2cids = _replacevoids(
self.args.additional_problem_def['lids2cids'])
# define dict from objects' attributes so as they are ordered for printing alignment
# self.args_dict = collections.OrderedDict(sorted(vars(self.args).items()))
return self.args
class params(object):
height_feature_extractor = 1024
width_feature_extractor = 2048
preserve_aspect_ratio = False
# Ntrain = 2975
tfrecords_path = _PATH_CITYS
Nb = 12
training_lids2cids = [
-1, 0, -1, -1, -1, -1, -1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, -1,
12, -1, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
]
training_problem_def = {'lids2cids': copy.copy(training_lids2cids)}
training_lids2cids = _replacevoids(training_lids2cids)
plotting = False
distribute = False
def _map_metrics_to_evaluation_problem_def(self, metrics):
# if a net should be evaluated with problem that is not the problem with which it was
# trained for, then the mappings from that problem should be provided.
# only confusion_matrix is suppported for now
assert set(metrics.keys()) == {
'global_step', 'loss', 'confusion_matrix'
}, ('internal error: only confusion matrix metric is supported for mapping to'
'a new problem definition for now. Change to training problem definition.'
)
assert 'training_cids2evaluation_cids' in self._settings.evaluation_problem_def.keys(
), ('Evaluation problem definition should have training_cids2evaluation_cids field.'
)
old_cm = metrics['confusion_matrix']
tcids2ecids = np.array(
_replacevoids(
self._settings.
evaluation_problem_def['training_cids2evaluation_cids']))
# TODO: confusion matrix type and shape assertions
assert old_cm.shape[0] == tcids2ecids.shape[
0], 'Mapping lengths should me equal. %i (old_cm) %i (tcids2ecids)' % \
(old_cm.shape[0], tcids2ecids.shape[0])
temp_shape = (max(tcids2ecids) + 1, old_cm.shape[1])
temp_cm = np.zeros(temp_shape, dtype=np.int64)
# mas noiazei to kathe kainourio apo poio palio pairnei:
# i row of the new cm takes from rows of the old cm with indices:from_indices
for i in range(temp_shape[0]):
from_indices = [k for k, x in enumerate(tcids2ecids) if x == i]
# print(from_indices)
for fi in from_indices:
temp_cm[i, :] += old_cm[fi, :].astype(np.int64)
# oi grammes athroistikan kai tora tha athroistoun kai oi stiles
new_shape = (max(tcids2ecids) + 1, max(tcids2ecids) + 1)
new_cm = np.zeros(new_shape, dtype=np.int64)
for j in range(new_shape[1]):
from_indices = [k for k, x in enumerate(tcids2ecids) if x == j]
# print(from_indices)
for fi in from_indices:
new_cm[:, j] += temp_cm[:, fi]
metrics['confusion_matrix'] = new_cm
return metrics
def _evaluate_preprocess(image, label, params):
_SIZE_FEATURE_EXTRACTOR = (params.height_network, params.width_network)
## prepare
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
label = tf.gather(tf.cast(_replacevoids(params.evaluation_problem_def['lids2cids']), tf.int32), tf.to_int32(label))
## preprocess
proimage = tf.image.resize_images(image, _SIZE_FEATURE_EXTRACTOR)
prolabel = tf.image.resize_images(label[..., tf.newaxis],
_SIZE_FEATURE_EXTRACTOR,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)[..., 0]
proimage, _ = preprocess_evaluate(proimage)
print('debug: proimage, prolabel', proimage, prolabel)
return image, label, proimage, prolabel
def _lids2cids(lids2cids, lids):
"""
Label ids to class ids conversion of ground truth using the lids2cids mapping.
This function gathers cids from lids2cids according to indices from lids.
Nl: number of labels
Nc: number of classes
Args:
lids2cids: Nl, in [0, Nc-1]
lids: H x W, tf.uint8, in [0, Nl-1]
Returns:
H x W, in [0, Nc-1]
"""
# TODO: add type checking
assert lids.dtype.is_integer, 'lids tensor must be integer.'
return tf.gather(_replacevoids(lids2cids), lids)
def _map_predictions_to_new_cids(predictions, old_cids2new_cids):
"""Map training predictions to predictions according to inference problem definition."""
# transform predictions with new class ids: decs and probs correspond to training class ids
# for plotting they should be transformed to new (inference or evaluation) class ids
# e.g. output_Nclasses = 5,
# training_cids2inference_cids = [-1, 1, 1, 0, -1] --> [2, 1, 1, 0, 2]
# before: probs.shape: (..., 5) --> after: probs.shape: (..., 3)
# for probs: using the probability of union rule: P(A ∪ B) = P(A) + P(B) - P(A ∩ B)
# with P(A ∩ B) = 0 probabilities have to be summed
# TODO(panos): save computations by checking if inference and training
# problem defs are the same
supported_keys = {
'l1_probabilities', 'rawimages', 'rawimagespaths', 'decisions',
'l1_decisions', 'l1_logits', 'l1_probabilities',
'l2_vehicle_decisions', 'l2_vehicle_logits',
'l2_vehicle_probabilities', 'l2_human_decisions', 'l2_human_logits',
'l2_human_probabilities'
}
# check if supported_keys is a superset of predictions.keys()
assert supported_keys >= predictions.keys(), (
f"Supported keys are {sorted(supported_keys)} and predictions keys are {sorted(set(predictions.keys()))}. "
f"Supported keys must be a superset of predictions keys for resizing predictions."
)
probs, decs = itemgetter('l1_probabilities', 'decisions')(predictions)
old_cids2new_cids = _replacevoids(old_cids2new_cids)
ocids2ncids = tf.cast(old_cids2new_cids, tf.int32)
decs = tf.gather(ocids2ncids, decs)
try:
probs_transposed = tf.transpose(probs, (3, 0, 1, 2))
probs_transformed = tf.unsorted_segment_sum(
probs_transposed, ocids2ncids,
tf.reduce_max(ocids2ncids) + 1)
probs = tf.transpose(probs_transformed, (1, 2, 3, 0))
except:
tf.logging.info(
'\nl1_probabilities are not transformed to new cids.\n')
tf.logging.info('\nOnly decisions were transformed to new cids.\n')
new_predictions = predictions
new_predictions.update({'l1_probabilities': probs, 'decisions': decs})
return new_predictions
def _evaluate_preprocess(image, label, params):
SIZE_FEATURE_EXTRACTOR = (params.height_feature_extractor, params.width_feature_extractor)
## prepare
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
evaluation_lids2cids = _replacevoids(params.evaluation_problem_def['lids2cids'])
label = tf.gather(tf.cast(evaluation_lids2cids, tf.int32), tf.to_int32(label))
## preprocess
proimage = tf.image.resize_images(image, SIZE_FEATURE_EXTRACTOR)
prolabel = tf.image.resize_images(label[..., tf.newaxis],
SIZE_FEATURE_EXTRACTOR,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)[..., 0]
proimage = from_0_1_to_m1_1(proimage)
print('debug: proimage, prolabel', proimage, prolabel)
return image, label, proimage, prolabel
def __init__(self, input_fns, model_fn, settings=None):
"""Constructs a SS instance... (More info to be added...)
Args:
input_fns: a dictionary containing 'train', 'eval' and 'predict' keys to corresponding
input functions. The input functions will be called by the respective functions of this
class with the following signature: (customconfig, params) and should
return a tuple of (features, labels) containing feature and label dictionaries with
the needed str-tf.Tensor pairs. (required keys will be added, for now check code...).
model_fn: model function for the fully convolutional semantic segmentation model chosen.
It will be called with the following signature: (mode, features, labels, config, params).
settings: an object containing all parsed parameters from command line as attributes.
[UNUSED ARGUMENTS FOR NOW]
Comments / design choices:
1) Everytime a method {train, predict, evaluate} of this object is called a new local
estimator is created with the desired properties saved in this class' members, specified
to the respective action. This choice is made purely for memory efficiency.
"""
assert settings is not None, ('settings must be provided for now.')
_validate_settings(settings)
self._input_fns = input_fns
self._model_fn = model_fn
self._settings = copy.deepcopy(settings)
self._estimator = None
# by convention class ids start at 0
number_of_training_classes = max(
self._settings.training_problem_def['lids2cids']) + 1
trained_with_void_class = -1 in self._settings.training_problem_def[
'lids2cids'] # -1 indicates a void class
self._settings.training_Nclasses = number_of_training_classes + trained_with_void_class
self._settings.training_lids2cids = _replacevoids(
self._settings.training_problem_def['lids2cids'])
# save to settings for external access
self._settings.eval_res_dir = make_evaluation_dir(settings.log_dir)
def _map_predictions_to_inference_problem_def(self, predictions):
assert 'training_cids2inference_cids' in self._settings.inference_problem_def.keys(
), ('Inference problem definition should have training_cids2inference_cids field, '
'since provided inference problem definition file is not the same as training '
'problem definition file.')
tcids2pcids = np.array(
_replacevoids(
self._settings.
inference_problem_def['training_cids2inference_cids']))
for prediction in predictions:
# only decisions is suppported for now
assert set(prediction.keys(
)).intersection(set(self._settings.predict_keys)) == {
'decisions', 'rawimages', 'rawimagespaths'
}, ('internal error: only decisions predict_key is supported for mapping to '
'a new problem definition for now. Change to training problem definition.'
)
old_decisions = prediction['decisions']
# TODO: add type and shape assertions
assert old_decisions.ndim == 2, f"internal error: decisions shape is {old_decisions.shape}."
new_decisions = tcids2pcids[old_decisions]
if np.any(np.equal(new_decisions, -1)):
log.debug(
'WARNING: -1 label exists in decisions, handle it properly externally.'
)
# raise NotImplementedError(
# 'void mapping in different inference problem def is not yet implemented.')
prediction['decisions'] = new_decisions
yield prediction
def define_estimator(mode, features, labels, model_fn, config, params):
"""Add documentation... More information at tf.Estimator class.
Assumptions:
features: a dict containing rawfeatures and profeatures
both: Nb x hf x wf x 3, tf.float32, in [0,1]
labels: a dict containing rawfeatures and profeatures
both: Nb x hf x wf, tf.int32, in [0,Nc-1]
Args:
features: First item returned by input_fn passed to train, evaluate, and predict.
labels: Second item returned by input_fn passed to train, evaluate, and predict.
mode: one of tf.estimator.ModeKeys.
config: a tf.estimator.RunConfig object...
parameters: a tf.train.HParams object...
...
"""
assert mode in _ALLOWED_MODES, (
'mode should be TRAIN, EVAL or PREDICT from tf.estimator.ModeKeys.')
assert params.name_feature_extractor in {
'resnet_v1_50', 'resnet_v1_101'
}, ('params must have name_feature_extractor attribute in resnet_v1_{50,101}.'
)
if params.name_feature_extractor == 'resnet_v1_101':
raise NotImplementedError(
'Use of resnet_v1_101 as base feature extractor is not yet implemented.'
)
# unpack features
rawimages = features['rawimages'] if 'rawimages' in features.keys(
) else None
rawimagespaths = features[
'rawimagespaths'] if 'rawimagespaths' in features.keys() else None
proimages = features['proimages']
prolabels = labels if labels else None
## build a fully convolutional model for semantic segmentation
# predictions refer to the training class ids
# for plotting of results (inference) or assessment, predictions should be transformed
# using `{inference, evaluation}_problem_def`s
_, _, predictions = model_fn(mode, proimages, prolabels, config, params)
# TODO(panos): assert that proimages and predictions have same spatial size
if mode == tf.estimator.ModeKeys.TRAIN:
# global step
global_step = tf.train.get_or_create_global_step()
# losses
with tf.variable_scope('losses'):
losses = define_losses(mode, predictions, prolabels, config,
params)
# exponential moving averages
# creates variables in checkpoint with name: 'emas/' + <variable_name> +
# {'ExponentialMovingAverage,Momentum}
# ex.: for 'classifier/logits/Conv/biases' it saves also
# 'emas/classifier/logits/Conv/biases/ExponentialMovingAverage'
# and 'emas/classifier/logits/Conv/biases/Momentum'
# create_train_op guarantees to run GraphKeys.UPDATE_OPS collection
# before total_loss in every step, but doesn't give any guarantee
# for running after some other op, and since ema need to be run
# after applying the gradients maybe this code needs checking
if params.ema_decay > 0:
with tf.variable_scope('exponential_moving_averages'):
#for mv in slim.get_model_variables():
# print('slim.model_vars:', mv.op.name)
ema = tf.train.ExponentialMovingAverage(
params.ema_decay,
num_updates=global_step,
zero_debias=True)
variables_to_ema = []
for mv in tf.model_variables():
if 'BatchNorm/moving' not in mv.name:
variables_to_ema.append(mv)
print(
f"\nFound {len(tf.model_variables())} variables, saving exponential "
f"moving averages for {len(variables_to_ema)} of them.\n")
maintain_ema_op = ema.apply(var_list=variables_to_ema)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, maintain_ema_op)
# create training operation
with tf.variable_scope('train_ops'):
# optimizer
optimizer = define_optimizer(global_step, params)
# training op
train_op = create_train_op(
losses['total'],
optimizer,
global_step=global_step,
# update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS),
summarize_gradients=False,
# transform_grads_fn=,
# gradient_multipliers=gradient_multipliers,
check_numerics=False,
)
# TODO: maybe parameterize it
training_hooks = [
_RunMetadataHook(params.log_dir,
every_n_iter=max(params.num_training_steps // 50,
params.save_checkpoints_steps))
]
# next two lines were added for distributed debugging
if params.distribute:
tower_context = tf.contrib.distribute.get_tower_context()
assert tower_context
print(
f"Tower {tower_context.tower_id}: _RunMetadataHook is not supported "
"yet for distributed training.")
training_hooks = []
replace_initializers(config, params)
summaries_data = {
'features': features,
'labels': labels,
'predictions': predictions,
'losses': losses,
'learning_rate': optimizer._learning_rate
} #pylint: disable=protected-access
scaffold = _define_scaffold(mode, config, params, summaries_data)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
train_op=train_op,
training_hooks=training_hooks,
scaffold=scaffold)
if mode == tf.estimator.ModeKeys.EVAL:
with tf.variable_scope('losses'):
losses = define_losses(mode, predictions, prolabels, config,
params)
# returns (variable, update_op)
# TF internal error/problem: _streaming_confusion_matrix internally casts
# labels and predictions to int64, and since we feed a dictionary, tensors are
# passed by reference leading them to change type, thus we send an identity
# confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
# tf.identity(prolabels),
# tf.identity(predictions['decisions']),
# params.output_Nclasses)
# l1_probs, decs = itemgetter('l1_probabilities', 'decisions')(predictions)
# create a new dict with the supported keys only
predictions = _map_predictions_to_new_cids(
predictions, params.training_cids2evaluation_cids)
if params.replace_voids:
predictions = _replace_voids(predictions, params)
# TODO(panos): confusion matrix expects prolabels and predictions to have the same shape
# this may not the case when preserve_aspect_ratio is set and this will give an error
if hasattr(params, 'preserve_aspect_ratio'):
if params.preserve_aspect_ratio:
raise NotImplementedError(
'evaluation with preserving aspect ratio is not implemented.'
)
predictions = _resize_predictions(predictions,
tf.shape(labels['prolabels'])[1:3],
params)
tcids2ecids = _replacevoids(params.training_cids2evaluation_cids)
confusion_matrix = metrics_impl._streaming_confusion_matrix( # pylint: disable=protected-access
labels['prolabels'],
predictions['decisions'],
# +1 due to convention of starting counting at 0
max(tcids2ecids) + 1)
# dict of metrics keyed by name with values tuples of (metric_tensor, update_op)
# TODO: add more semantic segmentation metrics
eval_metric_ops = {
'confusion_matrix':
(tf.to_int32(confusion_matrix[0]), confusion_matrix[1])
}
scaffold = _define_scaffold(mode, config, params)
estimator_spec = tf.estimator.EstimatorSpec(
mode,
predictions=predictions,
loss=losses['total'],
eval_metric_ops=eval_metric_ops,
scaffold=scaffold)
if mode == tf.estimator.ModeKeys.PREDICT:
# create a new dict with the supported keys only
l1_probs, l2_vehicle_probs, l2_human_probs, decs = itemgetter(
'l1_probabilities', 'l2_vehicle_probabilities',
'l2_human_probabilities', 'decisions')(predictions)
predictions = {
'l1_probabilities': l1_probs,
'l2_vehicle_probabilities': l2_vehicle_probs,
'l2_human_probabilities': l2_human_probs,
'decisions': decs
}
# workaround for connecting input pipeline outputs to system output
# TODO(panos): maybe from a system perspective makes more sense to have mapping and
# resizing in the system_factory
# since these are functions of the system and not the network/estimator
# new size defaults to provided values
# if at least one is None then new size is the arbitrary size of rawimage in each step
new_size = (params.height_system, params.width_system)
is_arbitrary = not all(new_size)
if is_arbitrary:
if rawimages is not None:
predictions['rawimages'] = rawimages
if rawimagespaths is not None:
predictions['rawimagespaths'] = rawimagespaths
new_size = tf.shape(predictions['rawimages'])[1:3]
predictions = _resize_predictions(predictions, new_size, params)
tf.logging.warn(
'Mapping of predictions to new cids is not implemented for now.')
# predictions = _map_predictions_to_new_cids(predictions, params.training_cids2inference_cids)
if params.replace_voids:
predictions = _replace_voids(predictions, params)
scaffold = _define_scaffold(mode, config, params)
estimator_spec = tf.estimator.EstimatorSpec(mode,
predictions=predictions,
scaffold=scaffold)
return estimator_spec