def bboxes_nms(scores, bboxes, nms_threshold=0.5, keep_top_k=200, scope=None): """Apply non-maximum selection to bounding boxes. In comparison to TF implementation, use classes information for matching. Should only be used on single-entries. Use batch version otherwise. Args: scores: N Tensor containing float scores. bboxes: N x 4 Tensor containing boxes coordinates. nms_threshold: Matching threshold in NMS algorithm; keep_top_k: Number of total object to keep after NMS. Return: classes, scores, bboxes Tensors, sorted by score. Padded with zero if necessary. """ with tf.name_scope(scope, 'bboxes_nms_single', [scores, bboxes]): # Apply NMS algorithm. shape = (bboxes.shape[0], 4) boxxes_n = [] shape1= (bboxes.shape[0]) ones_m = tf.ones(shape1, dtype=tf.float32) zeros_m = tf.zeros(shape1,dtype=tf.float32) boxxes_n = tf.stack([bboxes[:,0],zeros_m,bboxes[:,1],ones_m],axis=1) idxes = tf.image.non_max_suppression(boxxes_n, scores, keep_top_k, nms_threshold) #idxes = bbox_1d_temporal_nms(bboxes, scores, keep_top_k, nms_threshold) scores = tf.gather(scores, idxes) bboxes = tf.gather(bboxes, idxes) # Pad results. scores = tfe_tensors.pad_axis(scores, 0, keep_top_k, axis=0) bboxes = tfe_tensors.pad_axis(bboxes, 0, keep_top_k, axis=0) return scores, bboxes
def bboxes_nms(classes, scores, bboxes, nms_threshold=0.4, keep_top_k=200, scope=None): """Apply non-maximum selection to bounding boxes. In comparison to TF implementation, use classes information for matching. Should only be used on single-entries. Use batch version otherwise. Args: scores: N Tensor containing float scores. bboxes: N x 4 Tensor containing boxes coordinates. nms_threshold: Matching threshold in NMS algorithm; keep_top_k: Number of total object to keep after NMS. Return: classes, scores, bboxes Tensors, sorted by score. Padded with zero if necessary. """ with tf.name_scope(scope, 'bboxes_nms_single', [classes, scores, bboxes]): # Apply NMS algorithm. idxes = tf.image.non_max_suppression(bboxes, scores, keep_top_k, nms_threshold) scores = tf.gather(scores, idxes) bboxes = tf.gather(bboxes, idxes) classes = tf.gather(classes, idxes) # Pad results. scores = tfe_tensors.pad_axis(scores, 0, keep_top_k, axis=0) bboxes = tfe_tensors.pad_axis(bboxes, 0, keep_top_k, axis=0) classes = tfe_tensors.pad_axis(classes, 0, keep_top_k, axis=0) return classes, scores, bboxes
def bboxes_filter_min(self, scores, bboxes, top_k, minsize=0.03, scope=None): """Sort bounding boxes by decreasing order and keep only the top_k. If inputs are dictionnaries, assume every key is a different class. Assume a batch-type input. Args: scores: Batch x N Tensor/Dictionary containing float scores. bboxes: Batch x N x 4 Tensor/Dictionary containing boxes coordinates. top_k: Top_k boxes to keep. Return: scores, bboxes: Sorted Tensors/Dictionaries of shape Batch x Top_k x 1|4. """ # Dictionaries as inputs. if isinstance(scores, dict) or isinstance(bboxes, dict): with tf.name_scope(scope, 'bboxes_sort_dict'): d_scores = {} d_bboxes = {} for c in scores.keys(): s, b = self.bboxes_filter_min(scores[c], bboxes[c], top_k, minsize=minsize) d_scores[c] = s d_bboxes[c] = b return d_scores, d_bboxes # Tensors inputs. with tf.name_scope(scope, 'bboxes_filter_min', [scores, bboxes]): scores, bboxes = tf.squeeze(scores, 0), tf.squeeze(bboxes, 0) h = (bboxes[:, 2] - bboxes[:, 0]) w = (bboxes[:, 3] - bboxes[:, 1]) mask = tf.greater(w, minsize) mask = tf.logical_and(mask, tf.greater(h, minsize)) # Boolean masking... scores = tf.boolean_mask(scores, mask) bboxes = tf.boolean_mask(bboxes, mask) scores = tfe_tensors.pad_axis(scores, 0, top_k, axis=0) bboxes = tfe_tensors.pad_axis(bboxes, 0, top_k, axis=0) return tf.expand_dims(scores, axis=0), tf.expand_dims(bboxes, axis=0)
def bboxes_sort(scores, bboxes, top_k=400, scope=None): """Sort bounding boxes by decreasing order and keep only the top_k. If inputs are dictionnaries, assume every key is a different class. Assume a batch-type input. Args: scores: Batch x N Tensor/Dictionary containing float scores. bboxes: Batch x N x 4 Tensor/Dictionary containing boxes coordinates. top_k: Top_k boxes to keep. Return: scores, bboxes: Sorted Tensors/Dictionaries of shape Batch x Top_k x 1|4. """ # Dictionaries as inputs. if isinstance(scores, dict) or isinstance(bboxes, dict): with tf.name_scope(scope, 'bboxes_sort_dict'): d_scores = {} d_bboxes = {} for c in scores.keys(): s, b = bboxes_sort(scores[c], bboxes[c], top_k=top_k) d_scores[c] = s d_bboxes[c] = b return d_scores, d_bboxes # Tensors inputs. with tf.name_scope(scope, 'bboxes_sort', [scores, bboxes]): # Sort scores... scores, idxes = tf.nn.top_k(scores, k=top_k, sorted=True) # Trick to be able to use tf.gather: map for each element in the first dim. def fn_gather(bboxes, idxes): bb = tf.gather(bboxes, idxes) return [bb] r = tf.map_fn(lambda x: fn_gather(x[0], x[1]), [bboxes, idxes], dtype=[bboxes.dtype], parallel_iterations=10, back_prop=False, swap_memory=False, infer_shape=True) bboxes = r[0] return tfe_tensors.pad_axis(scores, 0, top_k, axis=1), tfe_tensors.pad_axis(bboxes, 0, top_k, axis=1)
def bboxes_nms(classes, scores, bboxes, nms_threshold=0.5, num_classes=21, pad_output=True, scope=None): """Apply non-maximum selection to bounding boxes. In comparison to TF implementation, use classes information for matching. Should only be used on single-entries. Use batch version otherwise. Args: classes, scores, bboxes: N (or Nx4) input Tensors; nms_threshold: Matching threshold in NMS algorithm; num_classes: Number of classes in the dataset; pad_output: Pad output to input size. Useful for batching. Return: classes, scores, bboxes Tensors, sorted by score. Padded with zero if necessary. """ with tf.name_scope(scope, 'bboxes_nms_single'): max_output_size = tfe_tensors.get_shape(classes)[-1] l_classes = [] l_scores = [] l_bboxes = [] # Apply NMS algorithm on every class. for i in range(1, num_classes): mask = tf.equal(classes, i) sub_scores = tf.boolean_mask(scores, mask) sub_bboxes = tf.boolean_mask(bboxes, mask) sub_classes = tf.boolean_mask(classes, mask) idxes = tf.image.non_max_suppression(sub_bboxes, sub_scores, max_output_size, nms_threshold) l_classes.append(tf.gather(sub_classes, idxes)) l_scores.append(tf.gather(sub_scores, idxes)) l_bboxes.append(tf.gather(sub_bboxes, idxes)) # Concat results. classes = tf.concat(tf.tuple(l_classes), axis=0) scores = tf.concat(tf.tuple(l_scores), axis=0) bboxes = tf.concat(tf.tuple(l_bboxes), axis=0) # Sort by the final results by score. scores, idxes = tf.nn.top_k(scores, k=tf.size(scores), sorted=True) classes = tf.gather(classes, idxes) bboxes = tf.gather(bboxes, idxes) # Pad outputs to initial size. Necessary if use for in batches... if pad_output: classes = tfe_tensors.pad_axis(classes, 0, max_output_size, axis=0) scores = tfe_tensors.pad_axis(scores, 0, max_output_size, axis=0) bboxes = tfe_tensors.pad_axis(bboxes, 0, max_output_size, axis=0) return classes, scores, bboxes
def nms_proc(scores, bboxes): # sort all the bboxes scores, idxes = tf.nn.top_k(scores, k = num_anchors, sorted = True) bboxes = tf.gather(bboxes, idxes) ymin = bboxes[:, 0] xmin = bboxes[:, 1] ymax = bboxes[:, 2] xmax = bboxes[:, 3] vol_anchors = (xmax - xmin) * (ymax - ymin) nms_mask = tf.cast(tf.ones_like(scores, dtype=tf.int8), tf.bool) keep_mask = tf.cast(tf.zeros_like(scores, dtype=tf.int8), tf.bool) def safe_divide(numerator, denominator): return tf.where(tf.greater(denominator, 0), tf.divide(numerator, denominator), tf.zeros_like(denominator)) def get_scores(bbox, nms_mask): # the inner square inner_ymin = tf.maximum(ymin, bbox[0]) inner_xmin = tf.maximum(xmin, bbox[1]) inner_ymax = tf.minimum(ymax, bbox[2]) inner_xmax = tf.minimum(xmax, bbox[3]) h = tf.maximum(inner_ymax - inner_ymin, 0.) w = tf.maximum(inner_xmax - inner_xmin, 0.) inner_vol = h * w this_vol = (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) if mode == 'union': union_vol = vol_anchors - inner_vol + this_vol elif mode == 'min': union_vol = tf.minimum(vol_anchors, this_vol) else: raise ValueError('unknown mode to use for nms.') return safe_divide(inner_vol, union_vol) * tf.cast(nms_mask, tf.float32) def condition(index, nms_mask, keep_mask): return tf.logical_and(tf.reduce_sum(tf.cast(nms_mask, tf.int32)) > 0, tf.less(index, keep_top_k)) def body(index, nms_mask, keep_mask): # at least one True in nms_mask indices = tf.where(nms_mask)[0][0] bbox = bboxes[indices] this_mask = tf.one_hot(indices, num_anchors, on_value=False, off_value=True, dtype=tf.bool) keep_mask = tf.logical_or(keep_mask, tf.logical_not(this_mask)) nms_mask = tf.logical_and(nms_mask, this_mask) nms_scores = get_scores(bbox, nms_mask) nms_mask = tf.logical_and(nms_mask, nms_scores < nms_threshold) return [index+1, nms_mask, keep_mask] index = 0 [index, nms_mask, keep_mask] = tf.while_loop(condition, body, [index, nms_mask, keep_mask]) return tfe_tensors.pad_axis(tf.boolean_mask(scores, keep_mask), 0, keep_top_k, axis=0), tfe_tensors.pad_axis(tf.boolean_mask(bboxes, keep_mask), 0, keep_top_k, axis=0)
def bboxes_nms(scores, bboxes, nms_threshold=0.5, keep_top_k=200, scope=None): """Apply non-maximum selection to bounding boxes. In comparison to TF implementation, use classes information for matching. Should only be used on single-entries. Use batch version otherwise. Args: scores: N Tensor containing float scores. bboxes: N x 4 Tensor containing boxes coordinates. nms_threshold: Matching threshold in NMS algorithm; keep_top_k: Number of total object to keep after NMS. Return: classes, scores, bboxes Tensors, sorted by score. Padded with zero if necessary. """ with tf.name_scope(scope, 'bboxes_nms_single', [scores, bboxes]): # Apply NMS algorithm. idxes = tf.image.non_max_suppression(bboxes, scores, keep_top_k, nms_threshold) scores = tf.gather(scores, idxes) bboxes = tf.gather(bboxes, idxes) # Pad results. scores = tfe_tensors.pad_axis(scores, 0, keep_top_k, axis=0) bboxes = tfe_tensors.pad_axis(bboxes, 0, keep_top_k, axis=0) return scores, bboxes