def _visl_proposals(self,
image,
num_proposals,
proposals,
top_k=100,
height=224,
width=224,
name='proposals'):
"""Visualize proposal results to the tensorboard.
Args:
image: A [batch, height, width, channels] float tensor,
ranging from 0 to 255.
num_proposals: A [batch] int tensor.
proposals: A [batch, max_num_proposals, 4] float tensor.
height: Height of the visualized image.
width: Width of the visualized image.
"""
with tf.name_scope('visl_proposals'):
image = tf.image.resize_images(image, [height, width])
image = tf.cast(image, tf.uint8)
image = plotlib.draw_rectangles(image,
boxes=proposals[:, :top_k, :],
color=plotlib.RED,
fontscale=1.0)
tf.summary.image(name, image, max_outputs=5)
def test_draw_rectangles(self):
image = tf.placeholder(tf.uint8, shape=[None, None, 3])
boxes = tf.placeholder(tf.float32, shape=[None, 4])
scores = tf.placeholder(tf.float32, shape=[None])
labels = tf.placeholder(tf.string, shape=[None])
canvas = plotlib.draw_rectangles(image=tf.expand_dims(image, axis=0),
boxes=tf.expand_dims(boxes, axis=0),
scores=tf.expand_dims(scores, axis=0),
labels=tf.expand_dims(labels, axis=0),
color=(0, 0, 255),
thickness=1,
fontscale=1.0)
with self.test_session() as sess:
canvas = sess.run(canvas[0],
feed_dict={
image:
np.zeros((480, 640, 3), dtype=np.uint8),
boxes: [[0.1, 0.25, 0.9, 0.75],
[0.2, 0.35, 0.8, 0.65]],
scores: [0.123, 0.456],
labels: ['bbox1', 'bbox2']
})
filename = _TMPDIR + "/draw_rectangles.png"
cv2.imwrite(filename, canvas[:, :, ::-1]) # RGB to BGR.
tf.logging.info("The image with rectangle is written to %s.", filename)
def _visl_proposals_top_k(self,
image,
num_proposals,
proposals,
proposal_scores,
proposal_labels=None,
top_k=5,
threshold=0.01,
height=224,
width=224,
name='midn'):
"""Visualize top proposal results to the tensorboard.
Args:
image: A [batch, height, width, channels] float tensor,
ranging from 0 to 255.
num_proposals: A [batch] int tensor.
proposals: A [batch, max_num_proposals, 4] float tensor.
proposal_scores: A [batch, max_num_proposals] float tensor.
proposal_labels: A [batch, max_num_proposals] float tensor.
height: Height of the visualized image.
width: Width of the visualized image.
"""
with tf.name_scope('visl_proposals'):
image = tf.image.resize_images(image, [height, width])
image = tf.cast(image, tf.uint8)
(top_k_boxes, top_k_scores,
top_k_labels) = model_utils.get_top_k_boxes_and_scores(
proposals, proposal_scores, proposal_labels, k=top_k)
proposal_scores = tf.where(proposal_scores > threshold,
proposal_scores,
-9999.0 * tf.ones_like(proposal_scores))
image = plotlib.draw_rectangles(image,
boxes=top_k_boxes,
scores=top_k_scores,
labels=top_k_labels,
color=plotlib.RED,
fontscale=1.0)
tf.summary.image(name, image, max_outputs=5)
def _visl_proposals(self,
image,
number_of_proposals,
proposals,
proposal_scores,
vocabulary_list,
height=448,
width=448):
"""Visualize proposal results to the tensorboard.
Args:
image: A [batch, height, width, channels] float tensor,
ranging from 0 to 255.
number_of_proposals: A [batch] int tensor.
proposals: A [batch, max_num_proposals, 4] float tensor.
proposal_scores: A [batch, max_num_proposals, num_classes] float tensor.
vocabulary_list: Names of the classes.
height: Height of the visualized image.
width: Width of the visualized image.
"""
proposal_labels = tf.gather(vocabulary_list,
tf.argmax(proposal_scores, axis=-1))
proposal_scores = tf.reduce_max(proposal_scores, axis=-1)
image = tf.cast(tf.image.resize_images(image, [height, width]),
tf.uint8)
top_k = 5
top_k_boxes, top_k_scores, top_k_labels = model_utils.get_top_k_boxes_and_scores(
proposals, tf.sigmoid(proposal_scores), proposal_labels, k=top_k)
image = plotlib.draw_rectangles(image,
boxes=top_k_boxes,
scores=top_k_scores,
labels=top_k_labels,
color=plotlib.RED)
tf.summary.image("proposals", image, max_outputs=5)
def _calc_proposal_scores(self,
image,
number_of_proposals,
proposals,
crop_size=28,
num_classes=20):
"""Calculates second stage logits.
Args:
image: A [batch, height, width, 3] float tensor.
number_of_proposals: A [batch] int tensor.
proposals: a [batch, max_num_proposals, 4] float tensor.
Returns:
proposal_scores: a [batch, max_num_proposals, num_classes] float tensor.
"""
options = self._model_proto
is_training = self._is_training
# Stack proposal boxes.
batch, max_num_proposals, _ = utils.get_tensor_shape(proposals)
box_ind = tf.expand_dims(tf.range(batch), axis=-1)
box_ind = tf.reshape(tf.tile(box_ind, [1, max_num_proposals]), [-1])
proposals_flattened = tf.reshape(proposals, [-1, 4])
# Crop image and predict using CNN, image_cropped
# shape = [ batch * max_num_proposals, crop_size, crop_size, 3].
image_cropped = tf.image.crop_and_resize(
image,
boxes=proposals_flattened,
box_ind=box_ind,
crop_size=[crop_size, crop_size])
preprocessed_image_cropped = model_utils.preprocess_image(
image_cropped, options.preprocess_method)
with tf.variable_scope(CAMVariableScopes.cnn, reuse=True):
feature_cropped = model_utils.vgg_16(preprocessed_image_cropped,
is_training=is_training
and options.cnn_trainable)
feature_cropped = slim.dropout(feature_cropped,
options.cnn_dropout_keep_prob,
is_training=is_training)
with tf.variable_scope(CAMVariableScopes.image_proj, reuse=True):
proposal_scores = self._project_image(
feature_cropped,
num_outputs=num_classes,
hyperparams=options.image_proj_hyperparams,
is_training=is_training)
proposal_scores = tf.squeeze(proposal_scores, [1, 2])
proposal_scores = tf.reshape(proposal_scores,
[batch, max_num_proposals, num_classes])
# Visualize the crops.
height = width = 112
patches = tf.image.resize_images(image_cropped, [height, width])
patch_labels = tf.reshape(
tf.gather(self._vocabulary_list, tf.argmax(proposal_scores,
axis=-1)), [-1, 1])
patch_scores = tf.reshape(tf.reduce_max(proposal_scores, axis=-1),
[-1, 1])
patches = plotlib.draw_rectangles(
tf.cast(patches, tf.uint8),
boxes=tf.tile(tf.constant([[[0.1, 0.1, 0.1, 0.1]]]),
[batch * max_num_proposals, 1, 1]),
scores=tf.sigmoid(patch_scores),
labels=patch_labels,
color=plotlib.RED,
fontscale=0.6)
patches = tf.reshape(patches,
[batch, max_num_proposals, height, width, 3])
visl_crops = tf.concat(tf.unstack(patches, axis=1), axis=2)
tf.summary.image("crops", visl_crops, max_outputs=5)
return proposal_scores
def _visl_class_activation_map_list(self,
image,
class_act_map_list,
anchors,
anchor_scores_list,
vocabulary_list,
height=224,
width=224):
"""Visualize class activation map to the tensorboard.
Args:
image: A [batch, height, width, channels] float tensor,
ranging from 0 to 255.
class_act_map_list: A list of class activation map, each of them is a
[batch, height, width, num_classes] float tensor.
anchors: A [batch, number_of_anchors, 4] float tensor, in normalized
coordinates.
anchor_scores_list: A list of [batch, number_of_anchors, num_classes]
float tensor.
vocabulary_list: Names of the classes.
height: Height of the visualized image.
width: Width of the visualized image.
"""
options = self._model_proto
batch, _, _, _ = utils.get_tensor_shape(image)
# Initialize text labels to attach to images.
vocabulary_list = [
tf.tile(tf.expand_dims(w, axis=0), [batch])
for w in vocabulary_list
]
if options.feature_extractor == cam_model_pb2.CAMModel.FRCNN:
text_list = [
tf.tile(tf.expand_dims('scale %.2lf' % (s), axis=0), [batch])
for s in self._input_scales
]
elif options.feature_extractor == cam_model_pb2.CAMModel.SSD:
text_list = [
tf.tile(
tf.expand_dims('layer %s' % (n.split('/')[-1]), axis=0),
[batch]) for n in self._cnn_feature_names
]
merge_v_fn = lambda x: tf.concat(x, axis=1)
merge_h_fn = lambda x: tf.concat(x, axis=2)
# Visualize heat map from different resolutions.
visl_list = []
for class_act_map, anchor_scores, text in zip(class_act_map_list,
anchor_scores_list,
text_list):
image_visl = plotlib.draw_caption(
tf.cast(tf.image.resize_images(image, [height * 2, width * 2]),
tf.uint8), text, (5, 5), plotlib.RED)
visl_list_at_i = []
for class_id, x in enumerate(
tf.unstack(tf.image.resize_images(class_act_map,
[height, width]),
axis=-1)):
class_name = vocabulary_list[class_id]
# Look for the top-k box.
top_k = 1
top_k_boxes, top_k_scores, _ = model_utils.get_top_k_boxes_and_scores(
anchors, anchor_scores[:, :, class_id], k=top_k)
# Draw class-related heat map.
x = plotlib.convert_to_heatmap(x, normalize=False)
x = tf.image.convert_image_dtype(x, tf.uint8)
x = plotlib.draw_caption(x,
class_name,
org=(0, 0),
color=plotlib.RED)
# Draw bounding box.
x = plotlib.draw_rectangles(x,
boxes=top_k_boxes,
scores=top_k_scores,
color=plotlib.BLACK)
image_visl = plotlib.draw_rectangles(image_visl,
boxes=top_k_boxes,
color=plotlib.GREEN)
visl_list_at_i.append(x)
half_size = len(visl_list_at_i) // 2
visl_list.append(
merge_h_fn([image_visl] + [
merge_v_fn([
merge_h_fn(visl_list_at_i[:half_size]),
merge_h_fn(visl_list_at_i[half_size:])
])
]))
tf.summary.image("image", merge_v_fn(visl_list), max_outputs=5)