def test_convert_to_heatmap(self):
image = tf.placeholder(tf.float32, shape=[None, None])
heatmap = plotlib.convert_to_heatmap(tf.expand_dims(image, 0),
normalize=True)
with self.test_session() as sess:
kernel = imgproc._py_gaussian_kernel(ksize=100)
kernel_heatmap = sess.run(heatmap[0], feed_dict={image: kernel})
kernel_heatmap = (kernel_heatmap * 255).astype(np.uint8)
filename = _TMPDIR + "/convert_to_heatmap.png"
cv2.imwrite(filename, kernel_heatmap[:, :, ::-1]) # RGB to BGR.
tf.logging.info("The kernel image is written to %s.", filename)
def visualize(self,
image,
saliency,
interpolation=tf.image.ResizeMethod.NEAREST_NEIGHBOR):
"""Visualizes images to tensorboard.
Args:
image: a [batch, height, width, channels] float tensor, in [0, 255].
saliency: a [batch, feature_height, feature_width] float tensor.
"""
(batch, height, width, channels) = utils.get_tensor_shape(image)
image = image / 255.0
heatmap = plotlib.convert_to_heatmap(saliency, normalize=True)
heatmap = tf.image.resize_images(heatmap, [height, width],
interpolation)
heatmap = plotlib.gaussian_filter(heatmap, ksize=32)
image = tf.maximum(0.0, tf.concat([image, heatmap], axis=2))
tf.summary.image("images", image, max_outputs=10)
def _visl_class_activation_map(self,
image,
class_activation_map,
height=224,
width=224):
"""Visualize class activation map to the tensorboard.
Args:
image: A [batch, height, width, channels] float tensor,
ranging from 0 to 255.
class_activation_map: A [batch, height, width, num_classes] float tensor.
height: Height of the visualized image.
width: Width of the visualized image.
"""
options = self._model_proto
batch, _, _, _ = utils.get_tensor_shape(image)
# Initialize text labels to attach to images.
vocabulary_list = [
tf.tile(tf.expand_dims(w, axis=0), [batch])
for w in self._vocabulary_list
]
merge_v_fn = lambda x: tf.concat(x, axis=1)
merge_h_fn = lambda x: tf.concat(x, axis=2)
# Visualize heat map from different resolutions.
image_visl = tf.cast(
tf.image.resize_images(image, [height * 2, width * 2]), tf.uint8)
min_v = tf.reduce_min(class_activation_map,
axis=[1, 2, 3],
keepdims=True)
max_v = tf.reduce_max(class_activation_map,
axis=[1, 2, 3],
keepdims=True)
class_activation_map = (class_activation_map - min_v) / (max_v - min_v)
visl_list_at_i = []
for class_id, x in enumerate(
tf.unstack(tf.image.resize_images(class_activation_map,
[height, width]),
axis=-1)):
class_name = vocabulary_list[class_id]
# Draw class-related heat map.
x = plotlib.convert_to_heatmap(x, normalize=False)
x = tf.image.convert_image_dtype(x, tf.uint8)
x = plotlib.draw_caption(x,
class_name,
org=(0, 0),
color=plotlib.RED)
visl_list_at_i.append(x)
half_size = len(visl_list_at_i) // 2
visl_image = merge_h_fn([image_visl] + [
merge_v_fn([
merge_h_fn(visl_list_at_i[:half_size]),
merge_h_fn(visl_list_at_i[half_size:])
])
])
tf.summary.image("heatmap", visl_image, max_outputs=5)
def _predict_labels(self, examples):
"""Builds tf graph for prediction.
Args:
examples: dict of input tensors keyed by name.
Returns:
predictions: dict of prediction results keyed by name.
"""
options = self._model_proto
is_training = self._is_training
# Extract input data fields.
(image, image_id, num_captions, caption_strings,
caption_lengths) = (examples[InputDataFields.image],
examples[InputDataFields.image_id],
examples[InputDataFields.num_captions],
examples[InputDataFields.caption_strings],
examples[InputDataFields.caption_lengths])
class_act_map_predictions = self._calc_class_act_map(examples)
(class_act_map,
logits) = (class_act_map_predictions[VOCPredictions.class_act_map],
class_act_map_predictions[VOCPredictions.logits])
# Load the vocabulary.
vocabulary_list = self._read_vocabulary(options.vocabulary_file)
tf.logging.info("Read a vocabulary with %i words.", len(vocabulary_list))
# Encode labels, shape=[batch, num_classes].
class_labels = self._encode_labels(num_captions, caption_strings,
caption_lengths, vocabulary_list)
# visualize
with tf.name_scope("visualize"):
image_vis = tf.cast(image, tf.uint8)
image_vis = plotlib.draw_caption(
image_vis,
tf.reduce_join(caption_strings[:, 0, :], axis=-1, separator=','),
org=(5, 5),
fontscale=1.0,
color=(255, 0, 0),
thickness=1)
image_vis = plotlib.draw_caption(
image_vis,
tf.gather(vocabulary_list, tf.argmax(logits, axis=-1)),
org=(5, 25),
fontscale=1.0,
color=(255, 0, 0),
thickness=1)
class_act_map_list = []
batch_size, height, width, _ = utils.get_tensor_shape(image_vis)
for i, x in enumerate(tf.unstack(class_act_map, axis=-1)):
x = plotlib.convert_to_heatmap(x, normalize=True, normalize_to=[-4, 4])
#x = tf.image.resize_images(x, [height, width], tf.image.ResizeMethod.NEAREST_NEIGHBOR)
x = tf.image.resize_images(x, [height, width])
x = imgproc.gaussian_filter(x, ksize=32)
x = tf.image.convert_image_dtype(x, tf.uint8)
x = plotlib.draw_caption(
x,
tf.tile(tf.expand_dims(vocabulary_list[i], axis=0), [batch_size]),
org=(5, 5),
fontscale=1.0,
color=(255, 0, 0),
thickness=1)
class_act_map_list.append(x)
tf.summary.image(
"image",
tf.concat([image_vis] + class_act_map_list, axis=2),
max_outputs=1)
predictions = {
VOCPredictions.image_id: image_id,
VOCPredictions.class_labels: class_labels,
VOCPredictions.class_act_map: class_act_map,
VOCPredictions.logits: logits,
}
return predictions
def _visl_class_activation_map_list(self,
image,
class_act_map_list,
anchors,
anchor_scores_list,
vocabulary_list,
height=224,
width=224):
"""Visualize class activation map to the tensorboard.
Args:
image: A [batch, height, width, channels] float tensor,
ranging from 0 to 255.
class_act_map_list: A list of class activation map, each of them is a
[batch, height, width, num_classes] float tensor.
anchors: A [batch, number_of_anchors, 4] float tensor, in normalized
coordinates.
anchor_scores_list: A list of [batch, number_of_anchors, num_classes]
float tensor.
vocabulary_list: Names of the classes.
height: Height of the visualized image.
width: Width of the visualized image.
"""
options = self._model_proto
batch, _, _, _ = utils.get_tensor_shape(image)
# Initialize text labels to attach to images.
vocabulary_list = [
tf.tile(tf.expand_dims(w, axis=0), [batch])
for w in vocabulary_list
]
if options.feature_extractor == cam_model_pb2.CAMModel.FRCNN:
text_list = [
tf.tile(tf.expand_dims('scale %.2lf' % (s), axis=0), [batch])
for s in self._input_scales
]
elif options.feature_extractor == cam_model_pb2.CAMModel.SSD:
text_list = [
tf.tile(
tf.expand_dims('layer %s' % (n.split('/')[-1]), axis=0),
[batch]) for n in self._cnn_feature_names
]
merge_v_fn = lambda x: tf.concat(x, axis=1)
merge_h_fn = lambda x: tf.concat(x, axis=2)
# Visualize heat map from different resolutions.
visl_list = []
for class_act_map, anchor_scores, text in zip(class_act_map_list,
anchor_scores_list,
text_list):
image_visl = plotlib.draw_caption(
tf.cast(tf.image.resize_images(image, [height * 2, width * 2]),
tf.uint8), text, (5, 5), plotlib.RED)
visl_list_at_i = []
for class_id, x in enumerate(
tf.unstack(tf.image.resize_images(class_act_map,
[height, width]),
axis=-1)):
class_name = vocabulary_list[class_id]
# Look for the top-k box.
top_k = 1
top_k_boxes, top_k_scores, _ = model_utils.get_top_k_boxes_and_scores(
anchors, anchor_scores[:, :, class_id], k=top_k)
# Draw class-related heat map.
x = plotlib.convert_to_heatmap(x, normalize=False)
x = tf.image.convert_image_dtype(x, tf.uint8)
x = plotlib.draw_caption(x,
class_name,
org=(0, 0),
color=plotlib.RED)
# Draw bounding box.
x = plotlib.draw_rectangles(x,
boxes=top_k_boxes,
scores=top_k_scores,
color=plotlib.BLACK)
image_visl = plotlib.draw_rectangles(image_visl,
boxes=top_k_boxes,
color=plotlib.GREEN)
visl_list_at_i.append(x)
half_size = len(visl_list_at_i) // 2
visl_list.append(
merge_h_fn([image_visl] + [
merge_v_fn([
merge_h_fn(visl_list_at_i[:half_size]),
merge_h_fn(visl_list_at_i[half_size:])
])
]))
tf.summary.image("image", merge_v_fn(visl_list), max_outputs=5)