def _visl_proposals(self, image, num_proposals, proposals, top_k=100, height=224, width=224, name='proposals'): """Visualize proposal results to the tensorboard. Args: image: A [batch, height, width, channels] float tensor, ranging from 0 to 255. num_proposals: A [batch] int tensor. proposals: A [batch, max_num_proposals, 4] float tensor. height: Height of the visualized image. width: Width of the visualized image. """ with tf.name_scope('visl_proposals'): image = tf.image.resize_images(image, [height, width]) image = tf.cast(image, tf.uint8) image = plotlib.draw_rectangles(image, boxes=proposals[:, :top_k, :], color=plotlib.RED, fontscale=1.0) tf.summary.image(name, image, max_outputs=5)
def test_draw_rectangles(self): image = tf.placeholder(tf.uint8, shape=[None, None, 3]) boxes = tf.placeholder(tf.float32, shape=[None, 4]) scores = tf.placeholder(tf.float32, shape=[None]) labels = tf.placeholder(tf.string, shape=[None]) canvas = plotlib.draw_rectangles(image=tf.expand_dims(image, axis=0), boxes=tf.expand_dims(boxes, axis=0), scores=tf.expand_dims(scores, axis=0), labels=tf.expand_dims(labels, axis=0), color=(0, 0, 255), thickness=1, fontscale=1.0) with self.test_session() as sess: canvas = sess.run(canvas[0], feed_dict={ image: np.zeros((480, 640, 3), dtype=np.uint8), boxes: [[0.1, 0.25, 0.9, 0.75], [0.2, 0.35, 0.8, 0.65]], scores: [0.123, 0.456], labels: ['bbox1', 'bbox2'] }) filename = _TMPDIR + "/draw_rectangles.png" cv2.imwrite(filename, canvas[:, :, ::-1]) # RGB to BGR. tf.logging.info("The image with rectangle is written to %s.", filename)
def _visl_proposals_top_k(self, image, num_proposals, proposals, proposal_scores, proposal_labels=None, top_k=5, threshold=0.01, height=224, width=224, name='midn'): """Visualize top proposal results to the tensorboard. Args: image: A [batch, height, width, channels] float tensor, ranging from 0 to 255. num_proposals: A [batch] int tensor. proposals: A [batch, max_num_proposals, 4] float tensor. proposal_scores: A [batch, max_num_proposals] float tensor. proposal_labels: A [batch, max_num_proposals] float tensor. height: Height of the visualized image. width: Width of the visualized image. """ with tf.name_scope('visl_proposals'): image = tf.image.resize_images(image, [height, width]) image = tf.cast(image, tf.uint8) (top_k_boxes, top_k_scores, top_k_labels) = model_utils.get_top_k_boxes_and_scores( proposals, proposal_scores, proposal_labels, k=top_k) proposal_scores = tf.where(proposal_scores > threshold, proposal_scores, -9999.0 * tf.ones_like(proposal_scores)) image = plotlib.draw_rectangles(image, boxes=top_k_boxes, scores=top_k_scores, labels=top_k_labels, color=plotlib.RED, fontscale=1.0) tf.summary.image(name, image, max_outputs=5)
def _visl_proposals(self, image, number_of_proposals, proposals, proposal_scores, vocabulary_list, height=448, width=448): """Visualize proposal results to the tensorboard. Args: image: A [batch, height, width, channels] float tensor, ranging from 0 to 255. number_of_proposals: A [batch] int tensor. proposals: A [batch, max_num_proposals, 4] float tensor. proposal_scores: A [batch, max_num_proposals, num_classes] float tensor. vocabulary_list: Names of the classes. height: Height of the visualized image. width: Width of the visualized image. """ proposal_labels = tf.gather(vocabulary_list, tf.argmax(proposal_scores, axis=-1)) proposal_scores = tf.reduce_max(proposal_scores, axis=-1) image = tf.cast(tf.image.resize_images(image, [height, width]), tf.uint8) top_k = 5 top_k_boxes, top_k_scores, top_k_labels = model_utils.get_top_k_boxes_and_scores( proposals, tf.sigmoid(proposal_scores), proposal_labels, k=top_k) image = plotlib.draw_rectangles(image, boxes=top_k_boxes, scores=top_k_scores, labels=top_k_labels, color=plotlib.RED) tf.summary.image("proposals", image, max_outputs=5)
def _calc_proposal_scores(self, image, number_of_proposals, proposals, crop_size=28, num_classes=20): """Calculates second stage logits. Args: image: A [batch, height, width, 3] float tensor. number_of_proposals: A [batch] int tensor. proposals: a [batch, max_num_proposals, 4] float tensor. Returns: proposal_scores: a [batch, max_num_proposals, num_classes] float tensor. """ options = self._model_proto is_training = self._is_training # Stack proposal boxes. batch, max_num_proposals, _ = utils.get_tensor_shape(proposals) box_ind = tf.expand_dims(tf.range(batch), axis=-1) box_ind = tf.reshape(tf.tile(box_ind, [1, max_num_proposals]), [-1]) proposals_flattened = tf.reshape(proposals, [-1, 4]) # Crop image and predict using CNN, image_cropped # shape = [ batch * max_num_proposals, crop_size, crop_size, 3]. image_cropped = tf.image.crop_and_resize( image, boxes=proposals_flattened, box_ind=box_ind, crop_size=[crop_size, crop_size]) preprocessed_image_cropped = model_utils.preprocess_image( image_cropped, options.preprocess_method) with tf.variable_scope(CAMVariableScopes.cnn, reuse=True): feature_cropped = model_utils.vgg_16(preprocessed_image_cropped, is_training=is_training and options.cnn_trainable) feature_cropped = slim.dropout(feature_cropped, options.cnn_dropout_keep_prob, is_training=is_training) with tf.variable_scope(CAMVariableScopes.image_proj, reuse=True): proposal_scores = self._project_image( feature_cropped, num_outputs=num_classes, hyperparams=options.image_proj_hyperparams, is_training=is_training) proposal_scores = tf.squeeze(proposal_scores, [1, 2]) proposal_scores = tf.reshape(proposal_scores, [batch, max_num_proposals, num_classes]) # Visualize the crops. height = width = 112 patches = tf.image.resize_images(image_cropped, [height, width]) patch_labels = tf.reshape( tf.gather(self._vocabulary_list, tf.argmax(proposal_scores, axis=-1)), [-1, 1]) patch_scores = tf.reshape(tf.reduce_max(proposal_scores, axis=-1), [-1, 1]) patches = plotlib.draw_rectangles( tf.cast(patches, tf.uint8), boxes=tf.tile(tf.constant([[[0.1, 0.1, 0.1, 0.1]]]), [batch * max_num_proposals, 1, 1]), scores=tf.sigmoid(patch_scores), labels=patch_labels, color=plotlib.RED, fontscale=0.6) patches = tf.reshape(patches, [batch, max_num_proposals, height, width, 3]) visl_crops = tf.concat(tf.unstack(patches, axis=1), axis=2) tf.summary.image("crops", visl_crops, max_outputs=5) return proposal_scores
def _visl_class_activation_map_list(self, image, class_act_map_list, anchors, anchor_scores_list, vocabulary_list, height=224, width=224): """Visualize class activation map to the tensorboard. Args: image: A [batch, height, width, channels] float tensor, ranging from 0 to 255. class_act_map_list: A list of class activation map, each of them is a [batch, height, width, num_classes] float tensor. anchors: A [batch, number_of_anchors, 4] float tensor, in normalized coordinates. anchor_scores_list: A list of [batch, number_of_anchors, num_classes] float tensor. vocabulary_list: Names of the classes. height: Height of the visualized image. width: Width of the visualized image. """ options = self._model_proto batch, _, _, _ = utils.get_tensor_shape(image) # Initialize text labels to attach to images. vocabulary_list = [ tf.tile(tf.expand_dims(w, axis=0), [batch]) for w in vocabulary_list ] if options.feature_extractor == cam_model_pb2.CAMModel.FRCNN: text_list = [ tf.tile(tf.expand_dims('scale %.2lf' % (s), axis=0), [batch]) for s in self._input_scales ] elif options.feature_extractor == cam_model_pb2.CAMModel.SSD: text_list = [ tf.tile( tf.expand_dims('layer %s' % (n.split('/')[-1]), axis=0), [batch]) for n in self._cnn_feature_names ] merge_v_fn = lambda x: tf.concat(x, axis=1) merge_h_fn = lambda x: tf.concat(x, axis=2) # Visualize heat map from different resolutions. visl_list = [] for class_act_map, anchor_scores, text in zip(class_act_map_list, anchor_scores_list, text_list): image_visl = plotlib.draw_caption( tf.cast(tf.image.resize_images(image, [height * 2, width * 2]), tf.uint8), text, (5, 5), plotlib.RED) visl_list_at_i = [] for class_id, x in enumerate( tf.unstack(tf.image.resize_images(class_act_map, [height, width]), axis=-1)): class_name = vocabulary_list[class_id] # Look for the top-k box. top_k = 1 top_k_boxes, top_k_scores, _ = model_utils.get_top_k_boxes_and_scores( anchors, anchor_scores[:, :, class_id], k=top_k) # Draw class-related heat map. x = plotlib.convert_to_heatmap(x, normalize=False) x = tf.image.convert_image_dtype(x, tf.uint8) x = plotlib.draw_caption(x, class_name, org=(0, 0), color=plotlib.RED) # Draw bounding box. x = plotlib.draw_rectangles(x, boxes=top_k_boxes, scores=top_k_scores, color=plotlib.BLACK) image_visl = plotlib.draw_rectangles(image_visl, boxes=top_k_boxes, color=plotlib.GREEN) visl_list_at_i.append(x) half_size = len(visl_list_at_i) // 2 visl_list.append( merge_h_fn([image_visl] + [ merge_v_fn([ merge_h_fn(visl_list_at_i[:half_size]), merge_h_fn(visl_list_at_i[half_size:]) ]) ])) tf.summary.image("image", merge_v_fn(visl_list), max_outputs=5)