def testOutputsToNumClasses(self):
num_classes = 21
model_options = common.ModelOptions(
outputs_to_num_classes={common.OUTPUT_TYPE: num_classes})
self.assertEqual(
model_options.outputs_to_num_classes[common.OUTPUT_TYPE],
num_classes)
def testForwardpassDeepLabv3plus(self):
crop_size = [33, 33]
outputs_to_num_classes = {'semantic': 3}
model_options = common.ModelOptions(
outputs_to_num_classes, crop_size,
output_stride=16)._replace(add_image_level_feature=True,
aspp_with_batch_norm=True,
logits_kernel_size=1,
decoder_output_stride=[4],
model_variant='mobilenet_v2'
) # Employ MobileNetv2 for fast test.
g = tf.Graph()
with g.as_default():
with self.test_session(graph=g) as sess:
inputs = tf.random_uniform((1, crop_size[0], crop_size[1], 3))
outputs_to_scales_to_logits = model.multi_scale_logits(
inputs, model_options, image_pyramid=[1.0])
sess.run(tf.global_variables_initializer())
outputs_to_scales_to_logits = sess.run(
outputs_to_scales_to_logits)
# Check computed results for each output type.
for output in outputs_to_num_classes:
scales_to_logits = outputs_to_scales_to_logits[output]
# Expect only one output.
self.assertEqual(len(scales_to_logits), 1)
for logits in scales_to_logits.values():
self.assertTrue(logits.any())
def testBuildDeepLabv2(self):
batch_size = 2
crop_size = [41, 41]
# Test with two image_pyramids.
image_pyramids = [[1], [0.5, 1]]
# Test two model variants.
model_variants = ['xception_65', 'mobilenet_v2']
# Test with two output_types.
outputs_to_num_classes = {'semantic': 3, 'direction': 2}
expected_endpoints = [['merged_logits'],
['merged_logits', 'logits_0.50', 'logits_1.00']]
expected_num_logits = [1, 3]
for model_variant in model_variants:
model_options = common.ModelOptions(
outputs_to_num_classes)._replace(
add_image_level_feature=False,
aspp_with_batch_norm=False,
aspp_with_separable_conv=False,
model_variant=model_variant)
for i, image_pyramid in enumerate(image_pyramids):
g = tf.Graph()
with g.as_default():
with self.test_session(graph=g):
inputs = tf.random_uniform(
(batch_size, crop_size[0], crop_size[1], 3))
outputs_to_scales_to_logits = model.multi_scale_logits(
inputs, model_options, image_pyramid=image_pyramid)
# Check computed results for each output type.
for output in outputs_to_num_classes:
scales_to_logits = outputs_to_scales_to_logits[
output]
self.assertListEqual(
sorted(scales_to_logits.keys()),
sorted(expected_endpoints[i]))
# Expected number of logits = len(image_pyramid) + 1, since the
# last logits is merged from all the scales.
self.assertEqual(len(scales_to_logits),
expected_num_logits[i])
def testDeepcopy(self):
num_classes = 21
model_options = common.ModelOptions(
outputs_to_num_classes={common.OUTPUT_TYPE: num_classes})
model_options_new = copy.deepcopy(model_options)
self.assertEqual(
(model_options_new.outputs_to_num_classes[common.OUTPUT_TYPE]),
num_classes)
num_classes_new = 22
model_options_new.outputs_to_num_classes[common.OUTPUT_TYPE] = (
num_classes_new)
self.assertEqual(
model_options.outputs_to_num_classes[common.OUTPUT_TYPE],
num_classes)
self.assertEqual(
(model_options_new.outputs_to_num_classes[common.OUTPUT_TYPE]),
num_classes_new)
def testBuildDeepLabWithDensePredictionCell(self):
batch_size = 1
crop_size = [33, 33]
outputs_to_num_classes = {'semantic': 2}
expected_endpoints = ['merged_logits']
dense_prediction_cell_config = [
{
'kernel': 3,
'rate': [1, 6],
'op': 'conv',
'input': -1
},
{
'kernel': 3,
'rate': [18, 15],
'op': 'conv',
'input': 0
},
]
model_options = common.ModelOptions(
outputs_to_num_classes, crop_size, output_stride=16)._replace(
aspp_with_batch_norm=True,
model_variant='mobilenet_v2',
dense_prediction_cell_config=dense_prediction_cell_config)
g = tf.Graph()
with g.as_default():
with self.test_session(graph=g):
inputs = tf.random_uniform(
(batch_size, crop_size[0], crop_size[1], 3))
outputs_to_scales_to_model_results = model.multi_scale_logits(
inputs, model_options, image_pyramid=[1.0])
for output in outputs_to_num_classes:
scales_to_model_results = outputs_to_scales_to_model_results[
output]
self.assertListEqual(list(scales_to_model_results),
expected_endpoints)
self.assertEqual(len(scales_to_model_results), 1)
def _build_deeplab(params, iterator, outputs_to_num_classes, ignore_label,
class_to_color):
"""Builds a clone of DeepLab.
Args:
iterator: An iterator of type tf.data.Iterator for images and labels.
outputs_to_num_classes: A map from output type to the number of classes. For
example, for the task of semantic segmentation with 21 semantic classes,
we would have outputs_to_num_classes['semantic'] = 21.
ignore_label: Ignore label.
:param NewDeeplabTrainParams params:
"""
samples = iterator.get_next()
samples[common.IMAGE].set_shape([
params.train_batch_size, params.train_crop_size[0],
params.train_crop_size[1], 3
])
# Add name to input and label nodes so we can add to summary.
samples[common.IMAGE] = tf.identity(samples[common.IMAGE],
name=common.IMAGE)
samples[common.LABEL] = tf.identity(samples[common.LABEL],
name=common.LABEL)
model_options = common.ModelOptions(
# params=params,
outputs_to_num_classes=outputs_to_num_classes,
crop_size=[int(sz) for sz in params.train_crop_size],
atrous_rates=params.atrous_rates,
output_stride=params.output_stride)
outputs_to_scales_to_logits = model.multi_scale_logits(
samples[common.IMAGE],
model_options=model_options,
image_pyramid=params.image_pyramid,
weight_decay=params.weight_decay,
is_training=True,
fine_tune_batch_norm=params.fine_tune_batch_norm,
nas_training_hyper_parameters={
'drop_path_keep_prob': params.drop_path_keep_prob,
'total_training_steps': params.train_steps,
})
# Add name to graph node so we can add to summary.
output_type_dict = outputs_to_scales_to_logits[common.OUTPUT_TYPE]
output_type_dict[model.MERGED_LOGITS_SCOPE] = tf.identity(
output_type_dict[model.MERGED_LOGITS_SCOPE], name=common.OUTPUT_TYPE)
for output, num_classes in six.iteritems(outputs_to_num_classes):
train_utils.add_softmax_cross_entropy_loss_for_each_scale(
outputs_to_scales_to_logits[output],
samples[common.LABEL],
num_classes,
ignore_label,
loss_weight=1.0,
upsample_logits=params.upsample_logits,
hard_example_mining_step=params.hard_example_mining_step,
top_k_percent_pixels=params.top_k_percent_pixels,
scope=output)
# Log the summary
_log_summaries(
samples[common.IMAGE],
samples[common.LABEL],
num_classes,
output_type_dict[model.MERGED_LOGITS_SCOPE],
save_summaries_images=params.save_summaries_images,
class_to_color=class_to_color,
)
def run(params):
"""
:param NewDeeplabVisParams params:
"""
paramparse.to_flags(tf.app.flags.FLAGS, params, verbose=0, allow_missing=1)
tf.logging.set_verbosity(tf.logging.INFO)
# paramparse.from_flags(FLAGS, to_clipboard=1)
print('reading input images from {}'.format(params.dataset_dir))
# Get dataset-dependent information.
dataset = data_generator.Dataset(dataset_name=params.dataset,
is_test=1,
split_name=params.vis_split,
dataset_dir=params.dataset_dir,
batch_size=params.vis_batch_size,
crop_size=params.vis_crop_size,
min_resize_value=params.min_resize_value,
max_resize_value=params.max_resize_value,
resize_factor=params.resize_factor,
model_variant=params.model_variant,
is_training=False,
should_shuffle=False,
should_repeat=False)
train_id_to_eval_id = None
if dataset.dataset_name == data_generator.get_cityscapes_dataset_name():
print('Cityscapes requires converting train_id to eval_id.')
train_id_to_eval_id = _CITYSCAPES_TRAIN_ID_TO_EVAL_ID
# Prepare for visualization.
os.makedirs(params.vis_logdir, exist_ok=1)
save_dir = os.path.join(params.vis_logdir,
_SEMANTIC_PREDICTION_SAVE_FOLDER)
# os.makedirs(save_dir, exist_ok=1)
raw_save_dir = os.path.join(params.vis_logdir,
_RAW_SEMANTIC_PREDICTION_SAVE_FOLDER)
stacked_save_dir = os.path.join(params.vis_logdir, 'stacked')
os.makedirs(raw_save_dir, exist_ok=1)
os.makedirs(stacked_save_dir, exist_ok=1)
print('Visualizing on set: {} with split: {}'.format(
params.dataset, params.vis_split))
classes, composite_classes = read_class_info(params.class_info_path)
class_to_color = {i: k[1] for i, k in enumerate(classes)}
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
# params=params,
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in params.vis_crop_size],
atrous_rates=params.atrous_rates,
output_stride=params.output_stride)
if tuple(params.eval_scales) == (1.0, ):
print('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options=model_options,
image_pyramid=params.image_pyramid)
else:
print('Performing multi-scale test.')
if params.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=params.eval_scales,
add_flipped_images=params.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
# if params.min_resize_value and params.max_resize_value:
# """
# foul buggy garbage since v1.12: https://github.com/tensorflow/tensorflow/issues/38694
# as is so typical of the gunky piece of crap that is tensorflow
# """
# # Only support batch_size = 1, since we assume the dimensions of original
# # image after tf.squeeze is [height, width, 3].
# assert params.vis_batch_size == 1
#
# # Reverse the resizing and padding operations performed in preprocessing.
# # First, we slice the valid regions (i.e., remove padded region) and then
# # we resize the predictions back.
# original_image = tf.squeeze(samples[common.ORIGINAL_IMAGE])
# original_image_shape = tf.shape(original_image)
# predictions = tf.slice(
# predictions,
# [0, 0, 0],
# [1, original_image_shape[0], original_image_shape[1]])
# resized_shape = tf.to_int32([tf.squeeze(samples[common.HEIGHT]),
# tf.squeeze(samples[common.WIDTH])])
# predictions = tf.squeeze(
# tf.image.resize_images(tf.expand_dims(predictions, 3),
# resized_shape,
# method=tf.image.ResizeMethod.BILINEAR,
# align_corners=True), 3)
tf.train.get_or_create_global_step()
if params.quantize_delay_step >= 0:
tf.contrib.quantize.create_eval_graph()
num_iteration = 0
max_num_iteration = params.max_number_of_iterations
checkpoints_iterator = tf.contrib.training.checkpoints_iterator(
params.checkpoint_dir, min_interval_secs=params.eval_interval_secs)
print('save_dir: {}'.format(save_dir))
print('raw_save_dir: {}'.format(raw_save_dir))
print('stacked_save_dir: {}'.format(stacked_save_dir))
for checkpoint_path in checkpoints_iterator:
num_iteration += 1
print('Starting visualization at ' +
time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
print('Visualizing with model {}'.format(checkpoint_path))
scaffold = tf.train.Scaffold(
init_op=tf.global_variables_initializer())
session_creator = tf.train.ChiefSessionCreator(
scaffold=scaffold,
master=params.master,
checkpoint_filename_with_path=checkpoint_path)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=None) as sess:
batch = 0
# image_id_offset = 0
while not sess.should_stop():
print('Visualizing batch {:d}'.format(batch + 1))
_process_batch(
params=params,
sess=sess,
images=samples[common.IMAGE],
original_images_resized=samples[common.ORIGINAL_IMAGE],
semantic_predictions=predictions,
image_names=samples[common.IMAGE_NAME],
image_heights=samples[common.HEIGHT],
image_widths=samples[common.WIDTH],
# image_id_offset=image_id_offset,
save_dir=save_dir,
raw_save_dir=raw_save_dir,
stacked_save_dir=stacked_save_dir,
class_to_color=class_to_color,
# train_id_to_eval_id=train_id_to_eval_id
)
# image_id_offset += params.vis_batch_size
batch += 1
print('Finished visualization at ' +
time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
if num_iteration >= max_num_iteration > 0:
break
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
dataset = data_generator.Dataset(
dataset_name=FLAGS.dataset,
split_name=FLAGS.eval_split,
dataset_dir=FLAGS.dataset_dir,
batch_size=FLAGS.eval_batch_size,
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
min_resize_value=FLAGS.min_resize_value,
max_resize_value=FLAGS.max_resize_value,
resize_factor=FLAGS.resize_factor,
model_variant=FLAGS.model_variant,
num_readers=2,
is_training=False,
should_shuffle=False,
should_repeat=False)
tf.gfile.MakeDirs(FLAGS.eval_logdir)
print('Evaluating on %s set', FLAGS.eval_split)
with tf.Graph().as_default():
samples = dataset.get_one_shot_iterator().get_next()
model_options = common.ModelOptions(
outputs_to_num_classes={
common.OUTPUT_TYPE: dataset.num_of_classes
},
crop_size=[int(sz) for sz in FLAGS.eval_crop_size],
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
# Set shape in order for tf.contrib.tfprof.model_analyzer to work properly.
samples[common.IMAGE].set_shape([
FLAGS.eval_batch_size,
int(FLAGS.eval_crop_size[0]),
int(FLAGS.eval_crop_size[1]), 3
])
if tuple(FLAGS.eval_scales) == (1.0, ):
print('Performing single-scale test.')
predictions = model.predict_labels(
samples[common.IMAGE],
model_options,
image_pyramid=FLAGS.image_pyramid)
else:
print('Performing multi-scale test.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
samples[common.IMAGE],
model_options=model_options,
eval_scales=FLAGS.eval_scales,
add_flipped_images=FLAGS.add_flipped_images)
predictions = predictions[common.OUTPUT_TYPE]
predictions = tf.reshape(predictions, shape=[-1])
labels = tf.reshape(samples[common.LABEL], shape=[-1])
weights = tf.to_float(tf.not_equal(labels, dataset.ignore_label))
# Set ignore_label regions to label 0, because metrics.mean_iou requires
# range of labels = [0, dataset.num_classes). Note the ignore_label regions
# are not evaluated since the corresponding regions contain weights = 0.
labels = tf.where(tf.equal(labels, dataset.ignore_label),
tf.zeros_like(labels), labels)
predictions_tag = 'miou'
for eval_scale in FLAGS.eval_scales:
predictions_tag += '_' + str(eval_scale)
if FLAGS.add_flipped_images:
predictions_tag += '_flipped'
# Define the evaluation metric.
miou, update_op = tf.metrics.mean_iou(predictions,
labels,
dataset.num_of_classes,
weights=weights)
tf.summary.scalar(predictions_tag, miou)
summary_op = tf.summary.merge_all()
summary_hook = tf.contrib.training.SummaryAtEndHook(
log_dir=FLAGS.eval_logdir, summary_op=summary_op)
hooks = [summary_hook]
num_eval_iters = None
if FLAGS.max_number_of_evaluations > 0:
num_eval_iters = FLAGS.max_number_of_evaluations
if FLAGS.quantize_delay_step >= 0:
tf.contrib.quantize.create_eval_graph()
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.
TRAINABLE_VARS_PARAMS_STAT_OPTIONS)
tf.contrib.tfprof.model_analyzer.print_model_analysis(
tf.get_default_graph(),
tfprof_options=tf.contrib.tfprof.model_analyzer.FLOAT_OPS_OPTIONS)
tf.contrib.training.evaluate_repeatedly(
master=FLAGS.master,
checkpoint_dir=FLAGS.checkpoint_dir,
eval_ops=[update_op],
max_number_of_evaluations=num_eval_iters,
hooks=hooks,
eval_interval_secs=FLAGS.eval_interval_secs)
def testWrongDeepLabVariant(self):
model_options = common.ModelOptions(
[])._replace(model_variant='no_such_variant')
with self.assertRaises(ValueError):
model._get_logits(images=[], model_options=model_options)
def main(unused_argv):
tf.logging.set_verbosity(tf.logging.INFO)
print('Prepare to export model to: %s', FLAGS.export_path)
with tf.Graph().as_default():
image, image_size, resized_image_size = _create_input_tensors()
model_options = common.ModelOptions(
outputs_to_num_classes={common.OUTPUT_TYPE: FLAGS.num_classes},
crop_size=FLAGS.crop_size,
atrous_rates=FLAGS.atrous_rates,
output_stride=FLAGS.output_stride)
if tuple(FLAGS.inference_scales) == (1.0, ):
print('Exported model performs single-scale inference.')
predictions = model.predict_labels(
image,
model_options=model_options,
image_pyramid=FLAGS.image_pyramid)
else:
print('Exported model performs multi-scale inference.')
if FLAGS.quantize_delay_step >= 0:
raise ValueError(
'Quantize mode is not supported with multi-scale test.')
predictions = model.predict_labels_multi_scale(
image,
model_options=model_options,
eval_scales=FLAGS.inference_scales,
add_flipped_images=FLAGS.add_flipped_images)
raw_predictions = tf.identity(
tf.cast(predictions[common.OUTPUT_TYPE], tf.float32),
_RAW_OUTPUT_NAME)
# Crop the valid regions from the predictions.
semantic_predictions = tf.slice(
raw_predictions, [0, 0, 0],
[1, resized_image_size[0], resized_image_size[1]])
# Resize back the prediction to the original image size.
def _resize_label(label, label_size):
# Expand dimension of label to [1, height, width, 1] for resize operation.
label = tf.expand_dims(label, 3)
resized_label = tf.image.resize_images(
label,
label_size,
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
align_corners=True)
return tf.cast(tf.squeeze(resized_label, 3), tf.int32)
semantic_predictions = _resize_label(semantic_predictions, image_size)
semantic_predictions = tf.identity(semantic_predictions,
name=_OUTPUT_NAME)
if FLAGS.quantize_delay_step >= 0:
tf.contrib.quantize.create_eval_graph()
saver = tf.train.Saver(tf.all_variables())
dirname = os.path.dirname(FLAGS.export_path)
tf.gfile.MakeDirs(dirname)
graph_def = tf.get_default_graph().as_graph_def(add_shapes=True)
freeze_graph.freeze_graph_with_def_protos(
graph_def,
saver.as_saver_def(),
FLAGS.checkpoint_path,
_OUTPUT_NAME,
restore_op_name=None,
filename_tensor_name=None,
output_graph=FLAGS.export_path,
clear_devices=True,
initializer_nodes=None)
if FLAGS.save_inference_graph:
tf.train.write_graph(graph_def, dirname, 'inference_graph.pbtxt')