def nms(dets, thresh, force_cpu=force_cpu): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: try: return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) except: return cpu_nms(dets, thresh) else: return cpu_nms(dets, thresh)
def nms_detection(dets, thresh): dets = np.ascontiguousarray(dets, dtype=np.float32) if cfg.USE_GPU: return gpu_nms(dets, thresh) return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=True): """Dispatch to CPU NMS implementations.""" if dets.shape[0] == 0: return [] return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): if dets.shape[0] == 0: return [] if force_cpu: return cpu_nms(dets, thresh) else: return gpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=True): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if not force_cpu: return gpu_nms(dets, thresh, device_id=0) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: from nms.gpu_nms import gpu_nms return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): if dets.shape[0] == 0: return [] if force_cpu: from nms.cpu_nms import cpu_nms return cpu_nms(dets, thresh) else: from nms.gpu_nms import gpu_nms return gpu_nms(dets, thresh)
def nms(dets, thresh): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS: return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=True): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] #if cfg.USE_GPU_NMS and not force_cpu: # return gpu_nms(dets, thresh, device_id=0) #else: return cpu_nms(dets, thresh)
def nms(dets, thresh, usegpu, gpu_id): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if usegpu: return gpu_nms(dets, thresh, device_id=gpu_id) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, is_merge=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS: return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, gpu=-1): """Dispatch to either CPU or GPU NMS implementations.""" # if dets.shape[0] == 0: # return [] # if gpu > -1: # return gpu_nms(dets, thresh, device_id=gpu) # else: return cpu_nms(dets, thresh)
def nms(dets, thresh, opts, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if opts.use_gpu_nms and not force_cpu: return gpu_nms(dets, thresh, device_id=opts.nms_gpu_id) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if not force_cpu: #return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) return gpu_nms(dets, thresh) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: GPU_ID = cfg.GPU_ID[0] if isinstance(cfg.GPU_ID, list) else cfg.GPU_ID return gpu_nms(dets, thresh, device_id=GPU_ID) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) else: print 'Warning from nms_wrapper.py: we use cpu-based nms function,sigh... ' return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] # print "gpu_id used by nms is: %d" % cfg.GPU_ID if cfg.USE_GPU_NMS and not force_cpu: return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: #print('use_gpu') return gpu_nms(dets, thresh, device_id=0) else: return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False, rectify=False, similarity=0.8): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: return gpu_nms(dets, thresh, device_id=cfg.GPU_ID) else: if rectify==False : return cpu_nms(dets, thresh) else : return cpu_re_nms(dets, thresh, similarity)
def nms(dets, thresh, force_cpu=False): """Dispatch to either CPU or GPU NMS implementations.""" if dets.shape[0] == 0: return [] if cfg.USE_GPU_NMS and not force_cpu: from nms.gpu_nms import gpu_nms return gpu_nms(dets, thresh, device_id=0) else: from nms.cpu_nms import cpu_nms return cpu_nms(dets, thresh)
def nms(self, scene, thresh): areas = scene['areas'].copy() if len(areas) < 1: return scene h = scene['height'] w = scene['width'] xywhs = scene['boxes'].copy() xyxys = xywhs_to_xyxys(xywhs, w, h) objs = np.concatenate([xyxys, areas[..., None]], -1).astype(np.float32) indices = np.array(cpu_nms(objs, thresh)) # if len(indices) < len(areas): # print(scene['image_index'], "%d --> %d"%(len(areas), len(indices))) return self.select_objects(scene, indices)
def __call__(self, rpn_cls_prob_reshape, rpn_bbox_pred, im_info): # Algorithm: # # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) assert rpn_cls_prob_reshape.shape[0] == 1, 'Only single item batches are supported' cfg_key = str(self.phase) pre_nms_topN = cfg[cfg_key].RPN_PRE_NMS_TOP_N post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N nms_thresh = cfg[cfg_key].RPN_NMS_THRESH min_size = cfg[cfg_key].RPN_MIN_SIZE max_size = 150 scores = rpn_cls_prob_reshape bbox_deltas = rpn_bbox_pred im_info = im_info[0, :] # 1. Generate proposals from bbox deltas and shifted anchors height, width = scores.shape[1:3] # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape((K, 1, 4)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # # bbox deltas will be (1, 4 * A, H, W) format # # transpose to (1, H, W, 4 * A) # bbox deltas is already in (1, H, W, 4 * A) format in tensorflow # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = bbox_deltas.reshape((-1, 4)) # Same story for the scores: # # # scores are (1, A, H, W) format # # transpose to (1, H, W, A) # scores is already in (1, H, W, A) format # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = scores.reshape((-1, 1)) # Convert anchors into proposals via bbox transformations proposals = bbox_transform_inv(anchors, bbox_deltas) # 2. clip predicted boxes to image proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = _filter_boxes(proposals, min_size * im_info[2], max_size) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) #nms_thresh = 0.1 keep = cpu_nms(np.hstack((proposals, scores)), nms_thresh) if post_nms_topN > 0: keep = keep[:post_nms_topN] proposals = proposals[keep, :] scores = scores[keep] # Output rois blob # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) return blob
def getBBoxesFromCAMs(CAMs, reshape_size=[256, 256], percentage_heat=0.4, size_restriction=0.1, box_expansion=0.2, use_gpu=True): ''' Reference: Bolaños, Marc, and Petia Radeva. "Simultaneous Food Localization and Recognition." arXiv preprint arXiv:1604.07953 (2016). Description: Extracts a set of bounding boxes from the generated CAMs which contain food instances. This functions should only be called if the current image has been predicted as Food by the GAP FoodvsNon-food detector! Arguments: :param CAMs: list of class activation maps generated by the CAM network :param reshape_size: reshape proportions used for transorming the CAM for extracting bounding boxes :param percentage_heat: minimum percentage allowed for considering a detection (aka 't' in reference paper) :param size_restriction: remove all regions covering less than a certain percentage size of the original image (aka 's' in reference paper) :param box_expansion: expand the bounding boxes by a certain percentage (aka 'e' in reference paper) :param use_gpu: boolean indicating if we want to use the GPU for applying NMS :return: [predicted_bboxes, predicted_scores], containing a list of bboxes coordinates on the first position and a list of their corresponding scores on the second position ''' from skimage.transform import resize from scipy import ndimage try: from nms.gpu_nms import gpu_nms from nms.cpu_nms import cpu_nms except: raise Exception( "Cython is required for running this function:\npip install cython\nRun the following command inside " "kernel_wrapper/extra/nms after its installation:\npython setup.py build_ext --inplace") predicted_bboxes = [] predicted_scores = [] # Get all computed maps (if we are also using convolutional features) all_maps = CAMs for map in all_maps: # map = misc.imread(maps_dir[dataset]+'/'+samples_detection[dataset]['all_ids'][s]+'_CAM.jpg') # CAM only # map = misc.imread(map_path) # CAM and convolutional features new_reshape_size = reshape_size # Resize map to original size map = resize(map, tuple(new_reshape_size), order=1, preserve_range=True) # Detect regions above a certain percentage of the max heat bb_thres = np.max(map) * percentage_heat # Compute binary selected region binary_heat = map binary_heat = np.where(binary_heat > bb_thres, 255, 0) # Get biggest connected component min_size = new_reshape_size[0] * new_reshape_size[1] * size_restriction labeled, nr_objects = ndimage.label(binary_heat) # get connected components [objects, counts] = np.unique(labeled, return_counts=True) # count occurrences biggest_components = np.argsort(counts[1:])[::-1] selected_components = [1 if counts[i + 1] >= min_size else 0 for i in biggest_components] # check minimum size restriction biggest_components = biggest_components[:min([np.sum(selected_components), 9999])] # get all bboxes # Extract each component (which will become a bbox prediction) map = map / 255.0 # normalize map # Get bboxes for selected, comp in zip(selected_components, biggest_components): if (selected): max_heat = np.where(labeled == comp + 1, 255, 0) # get the biggest # Draw bounding box on original image box = list(bbox(max_heat)) # expand box before final detection x_exp = box[2] * box_expansion y_exp = box[3] * box_expansion box[0] = max([0, box[0] - x_exp / 2]) box[1] = max([0, box[1] - y_exp / 2]) # change width and height by xmax and ymax box[2] += box[0] box[3] += box[1] box[2] = min([new_reshape_size[1] - 1, box[2] + x_exp]) box[3] = min([new_reshape_size[0] - 1, box[3] + y_exp]) predicted_bboxes.append(box) # Get score for current bbox score = np.mean(map[box[1]:box[3], box[0]:box[2]]) # use mean CAM value of the bbox as a score predicted_scores.append(score) # Now apply NMS on all the obtained bboxes nms_threshold = 0.3 # logging.info('bboxes before NMS: '+str(len(predicted_scores))) if (len(predicted_scores) > 0): dets = np.hstack((np.array(predicted_bboxes), np.array(predicted_scores)[:, np.newaxis])).astype(np.float32) if (use_gpu): keep = gpu_nms(dets, nms_threshold, device_id=0) else: keep = cpu_nms(dets, nms_threshold) dets = dets[keep, :] predicted_bboxes = [] predicted_scores = [] for idet in range(dets.shape[0]): predicted_bboxes.append(dets[idet, :4]) predicted_scores.append(dets[idet, -1]) # logging.info('bboxes after NMS: '+str(len(predicted_scores))) return [predicted_bboxes, predicted_scores]
def _nms(dets): return cpu_nms(dets, thresh)
def non_max_suppression(prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, labels=()): """Performs Non-Maximum Suppression (NMS) on inference results Returns: detections with shape: nx6 (x1, y1, x2, y2, conf, cls) """ nc = prediction.shape[2] - 5 # number of classes xc = prediction[..., 4] > conf_thres # candidates # Settings min_wh, max_wh = 2, 4096 # (pixels) minimum and maximum box width and height max_det = 300 # maximum number of detections per image max_nms = 30000 # maximum number of boxes into torchvision.ops.nms() time_limit = 10.0 # seconds to quit after redundant = True # require redundant detections multi_label = nc > 1 # multiple labels per box (adds 0.5ms/img) merge = False # use merge-NMS t = time.time() output = [torch.zeros((0, 6), device=prediction.device)] * prediction.shape[0] for xi, x in enumerate(prediction): # image index, image inference # Apply constraints # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height x = x[xc[xi]] # confidence # Cat apriori labels if autolabelling if labels and len(labels[xi]): l = labels[xi] v = torch.zeros((len(l), nc + 5), device=x.device) v[:, :4] = l[:, 1:5] # box v[:, 4] = 1.0 # conf v[range(len(l)), l[:, 0].long() + 5] = 1.0 # cls x = torch.cat((x, v), 0) # If none remain process next image if not x.shape[0]: continue # Compute conf x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf # Box (center x, center y, width, height) to (x1, y1, x2, y2) box = xywh2xyxy(x[:, :4]) # Detections matrix nx6 (xyxy, conf, cls) if multi_label: i, j = (x[:, 5:] > conf_thres).nonzero(as_tuple=False).T x = torch.cat((box[i], x[i, j + 5, None], j[:, None].float()), 1) else: # best class only conf, j = x[:, 5:].max(1, keepdim=True) x = torch.cat((box, conf, j.float()), 1)[conf.view(-1) > conf_thres] # Filter by class if classes is not None: x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] # Apply finite constraint # if not torch.isfinite(x).all(): # x = x[torch.isfinite(x).all(1)] # Check shape n = x.shape[0] # number of boxes if not n: # no boxes continue elif n > max_nms: # excess boxes x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence # Batched NMS # c = x[:, 5:6] * (0 if agnostic else max_wh) # classes # boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores dets = x[:,:5].cpu().detach().numpy() i=torch.as_tensor(cpu_nms(dets, iou_thres)) # i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS if i.shape[0] > max_det: # limit detections i = i[:max_det] if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix weights = iou * scores[None] # box weights x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes if redundant: i = i[iou.sum(1) > 1] # require redundancy output[xi] = x[i] if (time.time() - t) > time_limit: print(f'WARNING: NMS time limit {time_limit}s exceeded') break # time limit exceeded return output
def getBBoxesFromCAMs(CAMs, reshape_size=None, percentage_heat=0.4, size_restriction=0.1, box_expansion=0.2, use_gpu=True): """ Reference: Bolaños, Marc, and Petia Radeva. "Simultaneous Food Localization and Recognition." arXiv preprint arXiv:1604.07953 (2016). Description: Extracts a set of bounding boxes from the generated CAMs which contain food instances. This functions should only be called if the current image has been predicted as Food by the GAP FoodvsNon-food detector! Arguments: :param CAMs: list of class activation maps generated by the CAM network :param reshape_size: reshape proportions used for transorming the CAM for extracting bounding boxes :param percentage_heat: minimum percentage allowed for considering a detection (aka 't' in reference paper) :param size_restriction: remove all regions covering less than a certain percentage size of the original image (aka 's' in reference paper) :param box_expansion: expand the bounding boxes by a certain percentage (aka 'e' in reference paper) :param use_gpu: boolean indicating if we want to use the GPU for applying NMS :return: [predicted_bboxes, predicted_scores], containing a list of bboxes coordinates on the first position and a list of their corresponding scores on the second position """ from skimage.transform import resize from scipy import ndimage try: from nms.gpu_nms import gpu_nms from nms.cpu_nms import cpu_nms except: raise Exception( "Cython is required for running this function:\npip install cython\nRun the following command inside " "kernel_wrapper/extra/nms after its installation:\npython setup.py build_ext --inplace") if reshape_size is None: reshape_size = [256, 256] predicted_bboxes = [] predicted_scores = [] # Get all computed maps (if we are also using convolutional features) all_maps = CAMs for mapping in all_maps: # map = misc.imread(maps_dir[dataset]+'/'+samples_detection[dataset]['all_ids'][s]+'_CAM.jpg') # CAM only # map = misc.imread(map_path) # CAM and convolutional features new_reshape_size = reshape_size # Resize map to original size mapping = resize(mapping, tuple(new_reshape_size), order=1, preserve_range=True) # Detect regions above a certain percentage of the max heat bb_thres = np.max(mapping) * percentage_heat # Compute binary selected region binary_heat = mapping binary_heat = np.where(binary_heat > bb_thres, 255, 0) # Get biggest connected component min_size = new_reshape_size[0] * new_reshape_size[1] * size_restriction labeled, _ = ndimage.label(binary_heat) # get connected components [_, counts] = np.unique(labeled, return_counts=True) # count occurrences biggest_components = np.argsort(counts[1:])[::-1] selected_components = [1 if counts[i + 1] >= min_size else 0 for i in biggest_components] # check minimum size restriction biggest_components = biggest_components[:min([np.sum(selected_components), 9999])] # get all bboxes # Extract each component (which will become a bbox prediction) mapping = mapping / 255.0 # normalize map # Get bboxes for selected, comp in zip(selected_components, biggest_components): if (selected): max_heat = np.where(labeled == comp + 1, 255, 0) # get the biggest # Draw bounding box on original image box = list(bbox(max_heat)) # expand box before final detection x_exp = box[2] * box_expansion y_exp = box[3] * box_expansion box[0] = max([0, box[0] - x_exp / 2]) box[1] = max([0, box[1] - y_exp / 2]) # change width and height by xmax and ymax box[2] += box[0] box[3] += box[1] box[2] = min([new_reshape_size[1] - 1, box[2] + x_exp]) box[3] = min([new_reshape_size[0] - 1, box[3] + y_exp]) predicted_bboxes.append(box) # Get score for current bbox score = np.mean(mapping[box[1]:box[3], box[0]:box[2]]) # use mean CAM value of the bbox as a score predicted_scores.append(score) # Now apply NMS on all the obtained bboxes nms_threshold = 0.3 # logging.info('bboxes before NMS: '+str(len(predicted_scores))) if (len(predicted_scores) > 0): dets = np.hstack((np.array(predicted_bboxes), np.array(predicted_scores)[:, np.newaxis])).astype(np.float32) if (use_gpu): keep = gpu_nms(dets, nms_threshold, device_id=0) else: keep = cpu_nms(dets, nms_threshold) dets = dets[keep, :] predicted_bboxes = [] predicted_scores = [] for idet in range(dets.shape[0]): predicted_bboxes.append(dets[idet, :4]) predicted_scores.append(dets[idet, -1]) # logging.info('bboxes after NMS: '+str(len(predicted_scores))) return [predicted_bboxes, predicted_scores]