def _serving_model_graph(features, params): """Build the model graph for serving.""" model_outputs = mask_rcnn_model.build_model_graph( features, labels=None, is_training=False, params=params) if output_source_id: model_outputs.update({ 'source_id': features['source_id'], }) if output_image_info: model_outputs.update({ 'image_info': features['image_info'], }) final_boxes = model_outputs['detection_boxes'] if output_box_features: final_box_rois = model_outputs['detection_boxes'] final_roi_features = spatial_transform_ops.multilevel_crop_and_resize( model_outputs['fpn_feats'], final_box_rois, output_size=7) _, _, final_box_features = heads.box_head( final_roi_features, num_classes=params['num_classes'], mlp_head_dim=params['fast_rcnn_mlp_head_dim']) model_outputs.update({ 'detection_features': final_box_features, }) if output_normalized_coordinates: model_outputs['detection_boxes'] = box_utils.to_normalized_coordinates( final_boxes, tf.expand_dims(features['image_info'][:, 0], axis=-1), tf.expand_dims(features['image_info'][:, 1], axis=-1)) return model_outputs
def generate_detections_gpu(class_outputs, box_outputs, anchor_boxes, image_info, pre_nms_num_detections=1000, post_nms_num_detections=100, nms_threshold=0.3, bbox_reg_weights=(10., 10., 5., 5.)): """Generate the final detections given the model outputs (GPU version). Args: class_outputs: a tensor with shape [batch_size, N, num_classes], which stacks class logit outputs on all feature levels. The N is the number of total anchors on all levels. The num_classes is the number of classes predicted by the model. Note that the class_outputs here is the raw score. box_outputs: a tensor with shape [batch_size, N, num_classes*4], which stacks box regression outputs on all feature levels. The N is the number of total anchors on all levels. anchor_boxes: a tensor with shape [batch_size, N, 4], which stacks anchors on all feature levels. The N is the number of total anchors on all levels. image_info: a tensor of shape [batch_size, 5] which encodes each image's [height, width, scale, original_height, original_width]. pre_nms_num_detections: an integer that specifies the number of candidates before NMS. post_nms_num_detections: an integer that specifies the number of candidates after NMS. nms_threshold: a float number to specify the IOU threshold of NMS. bbox_reg_weights: a list of 4 float scalars, which are default weights on (dx, dy, dw, dh) for normalizing bbox regression targets. Returns: a tuple of tensors corresponding to number of valid boxes, box coordinates, object categories for each boxes, and box scores stacked in batch_size. """ with tf.name_scope('generate_detections'): batch_size, num_boxes, num_classes = class_outputs.get_shape().as_list( ) softmax_class_outputs = tf.nn.softmax(class_outputs) # Remove background scores = tf.slice(softmax_class_outputs, [0, 0, 1], [-1, -1, -1]) boxes = tf.slice( tf.reshape(box_outputs, [batch_size, num_boxes, num_classes, 4]), [0, 0, 1, 0], [-1, -1, -1, -1]) anchor_boxes = (tf.expand_dims(anchor_boxes, axis=2) * tf.ones([1, 1, num_classes - 1, 1])) num_detections = num_boxes * (num_classes - 1) boxes = tf.reshape(boxes, [batch_size, num_detections, 4]) scores = tf.reshape(scores, [batch_size, num_detections, 1]) anchor_boxes = tf.reshape(anchor_boxes, [batch_size, num_detections, 4]) # Decode boxes = box_utils.decode_boxes(boxes, anchor_boxes, bbox_reg_weights) # Clip boxes height = tf.expand_dims(image_info[:, 0:1], axis=-1) width = tf.expand_dims(image_info[:, 1:2], axis=-1) boxes = box_utils.clip_boxes(boxes, height, width) # NMS pre_nms_boxes = box_utils.to_normalized_coordinates( boxes, height, width) pre_nms_boxes = tf.reshape(pre_nms_boxes, [batch_size, num_boxes, num_classes - 1, 4]) pre_nms_scores = tf.reshape(scores, [batch_size, num_boxes, num_classes - 1]) (post_nms_boxes, post_nms_scores, post_nms_classes, post_nms_num_valid_boxes) = (tf.image.combined_non_max_suppression( pre_nms_boxes, pre_nms_scores, max_output_size_per_class=pre_nms_num_detections, max_total_size=post_nms_num_detections, iou_threshold=nms_threshold, score_threshold=0.0, pad_per_class=False)) post_nms_classes = post_nms_classes + 1 post_nms_boxes = box_utils.to_absolute_coordinates( post_nms_boxes, height, width) return (post_nms_num_valid_boxes, post_nms_boxes, tf.to_float(post_nms_classes), post_nms_scores)
def _propose_rois_gpu(scores, boxes, anchor_boxes, height, width, scale, rpn_pre_nms_topn, rpn_post_nms_topn, rpn_nms_threshold, rpn_min_size, bbox_reg_weights): """Proposes RoIs giva group of candidates (GPU version). Args: scores: a tensor with a shape of [batch_size, num_boxes]. boxes: a tensor with a shape of [batch_size, num_boxes, 4], in the encoded form. anchor_boxes: an Anchors object that contains the anchors with a shape of [batch_size, num_boxes, 4]. height: a tensor of shape [batch_size, 1, 1] representing the image height. width: a tensor of shape [batch_size, 1, 1] representing the image width. scale: a tensor of shape [batch_size, 1, 1] representing the image scale. rpn_pre_nms_topn: a integer number of top scoring RPN proposals to keep before applying NMS. This is *per FPN level* (not total). rpn_post_nms_topn: a integer number of top scoring RPN proposals to keep after applying NMS. This is the total number of RPN proposals produced. rpn_nms_threshold: a float number between 0 and 1 as the NMS threshold used on RPN proposals. rpn_min_size: a integer number as the minimum proposal height and width as both need to be greater than this number. Note that this number is at origingal image scale; not scale used during training or inference). bbox_reg_weights: None or a list of four integer specifying the weights used when decoding the box. Returns: scores: a tensor with a shape of [batch_size, rpn_post_nms_topn, 1] representing the scores of the proposals. It has same dtype as input scores. boxes: a tensor with a shape of [batch_size, rpn_post_nms_topn, 4] represneting the boxes of the proposals. The boxes are in normalized coordinates with a form of [ymin, xmin, ymax, xmax]. It has same dtype as input boxes. """ batch_size, num_boxes = scores.get_shape().as_list() topk_limit = min(num_boxes, rpn_pre_nms_topn) boxes = box_utils.decode_boxes(boxes, anchor_boxes, bbox_reg_weights) boxes = box_utils.clip_boxes(boxes, height, width) if rpn_min_size > 0.0: boxes, scores = box_utils.filter_boxes(boxes, tf.expand_dims(scores, axis=-1), rpn_min_size, height, width, scale) scores = tf.squeeze(scores, axis=-1) post_nms_topk_limit = (topk_limit if topk_limit < rpn_post_nms_topn else rpn_post_nms_topn) if rpn_nms_threshold > 0: # Normalize coordinates as combined_non_max_suppression currently # only support normalized coordinates. pre_nms_boxes = box_utils.to_normalized_coordinates( boxes, height, width) pre_nms_boxes = tf.reshape(pre_nms_boxes, [batch_size, num_boxes, 1, 4]) pre_nms_scores = tf.reshape(scores, [batch_size, num_boxes, 1]) boxes, scores, _, _ = tf.image.combined_non_max_suppression( pre_nms_boxes, pre_nms_scores, max_output_size_per_class=topk_limit, max_total_size=post_nms_topk_limit, iou_threshold=rpn_nms_threshold, score_threshold=0.0, pad_per_class=False) boxes = box_utils.to_absolute_coordinates(boxes, height, width) else: scores, boxes = box_utils.top_k(scores, k=post_nms_topk_limit, boxes_list=[boxes]) boxes = boxes[0] return scores, boxes
def generate_detections_gpu(class_outputs, box_outputs, anchor_boxes, image_id, image_info, pre_nms_num_detections=1000, post_nms_num_detections=100, nms_threshold=0.3, bbox_reg_weights=(10., 10., 5., 5.)): """Generate the final detections given the model outputs (GPU version). Args: class_outputs: a tensor with shape [batch_size, N, num_classes], which stacks class logit outputs on all feature levels. The N is the number of total anchors on all levels. The num_classes is the number of classes predicted by the model. Note that the class_outputs here is the raw score. box_outputs: a tensor with shape [batch_size, N, num_classes*4], which stacks box regression outputs on all feature levels. The N is the number of total anchors on all levels. anchor_boxes: a tensor with shape [batch_size, N, 4], which stacks anchors on all feature levels. The N is the number of total anchors on all levels. image_id: a tensor with shape [batch_size] which specifies the image id of each image in the batch. image_info: a tensor of shape [batch_size, 5] which encodes each image's [height, width, scale, original_height, original_width]. pre_nms_num_detections: an integer that specifies the number of candidates before NMS. post_nms_num_detections: an integer that specifies the number of candidates after NMS. nms_threshold: a float number to specify the IOU threshold of NMS. bbox_reg_weights: a list of 4 float scalars, which are default weights on (dx, dy, dw, dh) for normalizing bbox regression targets. Returns: detections: a tensor of [batch_size, post_nms_num_detections, 7], which stacks `post_nms_num_detections` number of detection results for each image in the batch. Each detection is stored in the format of [image_id, ymin, xmin, ymax, xmax, score, class] in the last dimension. """ batch_size, num_boxes, num_classes = class_outputs.get_shape().as_list() softmax_class_outputs = tf.nn.softmax(class_outputs) # Remove background scores = tf.slice(softmax_class_outputs, [0, 0, 1], [-1, -1, -1]) boxes = tf.slice( tf.reshape(box_outputs, [batch_size, num_boxes, num_classes, 4]), [0, 0, 1, 0], [-1, -1, -1, -1]) anchor_boxes = tf.tile(tf.expand_dims(anchor_boxes, axis=2), [1, 1, num_classes - 1, 1]) num_detections = num_boxes * (num_classes - 1) boxes = tf.reshape(boxes, [batch_size, num_detections, 4]) scores = tf.reshape(scores, [batch_size, num_detections, 1]) anchor_boxes = tf.reshape(anchor_boxes, [batch_size, num_detections, 4]) # Decode boxes = box_utils.decode_boxes(boxes, anchor_boxes, bbox_reg_weights) # Clip boxes height, width, scale = tf.split(image_info[:, :3], num_or_size_splits=3, axis=-1) height = tf.expand_dims(height, axis=-1) width = tf.expand_dims(width, axis=-1) scale = tf.expand_dims(scale, axis=-1) boxes = box_utils.clip_boxes(boxes, height, width) pre_nms_boxes = tf.reshape(boxes, [batch_size, num_boxes, num_classes - 1, 4]) pre_nms_scores = tf.reshape(scores, [batch_size, num_boxes, num_classes - 1]) # NMS pre_nms_boxes = box_utils.to_normalized_coordinates( pre_nms_boxes, height, width) post_nms_boxes, post_nms_scores, post_nms_classes, valid_boxes = ( tf.image.combined_non_max_suppression( pre_nms_boxes, pre_nms_scores, max_output_size_per_class=pre_nms_num_detections, max_total_size=post_nms_num_detections, iou_threshold=nms_threshold, score_threshold=0.0, pad_per_class=False)) post_nms_classes = post_nms_classes + 1 post_nms_boxes = box_utils.to_absolute_coordinates(post_nms_boxes, height, width) # Only works with static batch size. # Unroll batch dimension. post_boxes_list = tf.unstack(post_nms_boxes) post_scores_list = tf.unstack(post_nms_scores) post_classes_list = tf.unstack(post_nms_classes) valid_boxes_list = tf.unstack(valid_boxes) image_id_list = tf.unstack(image_id) detections = [] for boxes_i, scores_i, classes_i, _, image_id_i in (zip( post_boxes_list, post_scores_list, post_classes_list, valid_boxes_list, image_id_list)): post_nms_top_k_scores = tf.reshape(scores_i, [post_nms_num_detections]) post_nms_top_k_boxes = tf.reshape(boxes_i, [post_nms_num_detections, 4]) post_nms_top_k_classes = tf.reshape(classes_i, [post_nms_num_detections]) this_batch_detections = tf.stack([ tf.to_float(tf.fill(tf.shape(post_nms_top_k_scores), image_id_i)), post_nms_top_k_boxes[:, 0], post_nms_top_k_boxes[:, 1], post_nms_top_k_boxes[:, 2], post_nms_top_k_boxes[:, 3], post_nms_top_k_scores, tf.to_float(post_nms_top_k_classes), ], axis=1) detections.append(this_batch_detections) detections = tf.stack(detections, axis=0) return detections