def nms_detection(dets, thresh):
dets = np.ascontiguousarray(dets, dtype=np.float32)
if cfg.USE_GPU:
return gpu_nms(dets, thresh)
return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False):
if dets.shape[0] == 0:
return []
if force_cpu:
return cpu_nms(dets, thresh)
else:
return gpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if cfg.USE_GPU_NMS and not force_cpu:
return gpu_nms(dets, thresh, device_id=cfg.GPU_ID)
else:
return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=True):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if not force_cpu:
return gpu_nms(dets, thresh, device_id=0)
else:
return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if force_cpu:
return cpu_nms(dets, thresh)
return gpu_nms(dets, thresh)
def nms(dets, thresh):
if dets.shape[0] == 0:
return []
new_dets = np.copy(dets)
new_dets[:, 2] += new_dets[:, 0]
new_dets[:, 3] += new_dets[:, 1]
return gpu_nms(new_dets, thresh, device_id=GPU_ID)
def nms(dets, thresh, usegpu,gpu_id):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if usegpu:
return gpu_nms(dets, thresh, device_id=gpu_id)
else:
return cpu_nms(dets, thresh)
def nms(dets, thresh, opts, force_cpu=False):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if opts.use_gpu_nms and not force_cpu:
return gpu_nms(dets, thresh, device_id=opts.nms_gpu_id)
else:
return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if cfg.USE_GPU_NMS and not force_cpu:
return gpu_nms(dets, thresh, device_id=cfg.GPU_ID)
else:
return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False):
if dets.shape[0] == 0:
return []
if force_cpu:
from nms.cpu_nms import cpu_nms
return cpu_nms(dets, thresh)
else:
from nms.gpu_nms import gpu_nms
return gpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if cfg.USE_GPU_NMS and not force_cpu:
return gpu_nms(dets, thresh, device_id=cfg.GPU_ID)
else:
print 'Warning from nms_wrapper.py: we use cpu-based nms function,sigh... '
return cpu_nms(dets, thresh)
def nms(dets, thresh, force_cpu=False, rectify=False, similarity=0.8):
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if cfg.USE_GPU_NMS and not force_cpu:
return gpu_nms(dets, thresh, device_id=cfg.GPU_ID)
else:
if rectify==False : return cpu_nms(dets, thresh)
else : return cpu_re_nms(dets, thresh, similarity)
def nms(dets, thresh, force_cpu=False):
from nms.cpu_nms import cpu_nms, cpu_soft_nms
from nms.gpu_nms import gpu_nms
"""Dispatch to either CPU or GPU NMS implementations."""
if dets.shape[0] == 0:
return []
if force_cpu:
return cpu_soft_nms(dets, thresh, method=1)
#return cpu_nms(dets, thresh)
return gpu_nms(dets, thresh)
def nms_test_time(boxes_new):
thresh = [0.7,0.8,0.9]
T = 50
for i in range(len(thresh)):
since = time.time()
for t in range(T):
# keep = py_cpu_nms(boxes_new, thresh=thresh[i]) # for cpu
keep = gpu_nms(boxes_new, thresh=thresh[i]) # for gpu
print("thresh={:.1f}, time wastes:{:.4f}".format(thresh[i], (time.time()-since)/T))
return keep
def apply_nms(cls_dets, nms_method, nms_thresh):
# nms and filter
keep = np.where((cls_dets[:, 4] >= min_scores) &
((cls_dets[:, 3] - cls_dets[:, 1]) * (cls_dets[:, 2] - cls_dets[:, 0]) >= min_box_size))[0]
cls_dets = cls_dets[keep]
if len(cls_dets) > 0:
if nms_method == 'nms':
keep = gpu_nms(cls_dets, nms_thresh)
elif nms_method == 'soft':
keep = cpu_soft_nms(np.ascontiguousarray(cls_dets, dtype=np.float32), method=2)
else:
assert False
cls_dets = cls_dets[keep]
return cls_dets, keep
def nms_bboxes_dict(bboxes_dict):
for kfb_name, bboxes in bboxes_dict.items():
bboxes = np.array(bboxes).astype(np.float32)
keep = gpu_nms(bboxes, NMS_THRESH)
bboxes_dict[kfb_name] = []
for bbox in bboxes[keep]:
bbox_new = dict()
bbox_new['x'] = int(bbox[0])
bbox_new['y'] = int(bbox[1])
bbox_new['w'] = max(int(bbox[2] - bbox[0]), 1)
bbox_new['h'] = max(int(bbox[3] - bbox[1]), 1)
bbox_new['p'] = float(bbox[4])
bboxes_dict[kfb_name].append(bbox_new)
return bboxes_dict
def nms_test_time4(boxes_new):
from nms.gpu_nms import gpu_nms # 这里nms_py2是c源码文件,但在cython的pyx文件中优化了变量静态类型
thresh = [0.7, 0.8, 0.9]
T = 50
for i in range(len(thresh)):
since = time.time()
for t in range(T):
keep = gpu_nms(boxes_new, thresh=thresh[i]) # for cpu
# keep = gpu_nms(boxes_new, thresh=thresh[i]) # for gpu
print("thresh={:.1f}, time wastes:{:.4f}".format(
thresh[i], (time.time() - since) / T))
return keep
def getBBoxesFromCAMs(CAMs, reshape_size=None, percentage_heat=0.4, size_restriction=0.1, box_expansion=0.2,
use_gpu=True):
"""
Reference:
Bolaños, Marc, and Petia Radeva. "Simultaneous Food Localization and Recognition." arXiv preprint arXiv:1604.07953 (2016).
Description:
Extracts a set of bounding boxes from the generated CAMs which contain food instances.
This functions should only be called if the current image has been predicted as Food by the GAP FoodvsNon-food detector!
Arguments:
:param CAMs: list of class activation maps generated by the CAM network
:param reshape_size: reshape proportions used for transorming the CAM for extracting bounding boxes
:param percentage_heat: minimum percentage allowed for considering a detection (aka 't' in reference paper)
:param size_restriction: remove all regions covering less than a certain percentage size of the original image (aka 's' in reference paper)
:param box_expansion: expand the bounding boxes by a certain percentage (aka 'e' in reference paper)
:param use_gpu: boolean indicating if we want to use the GPU for applying NMS
:return: [predicted_bboxes, predicted_scores], containing a list of bboxes coordinates on the first position
and a list of their corresponding scores on the second position
"""
from skimage.transform import resize
from scipy import ndimage
try:
from nms.gpu_nms import gpu_nms
from nms.cpu_nms import cpu_nms
except:
raise Exception(
"Cython is required for running this function:\npip install cython\nRun the following command inside "
"kernel_wrapper/extra/nms after its installation:\npython setup.py build_ext --inplace")
if reshape_size is None:
reshape_size = [256, 256]
predicted_bboxes = []
predicted_scores = []
# Get all computed maps (if we are also using convolutional features)
all_maps = CAMs
for mapping in all_maps:
# map = misc.imread(maps_dir[dataset]+'/'+samples_detection[dataset]['all_ids'][s]+'_CAM.jpg') # CAM only
# map = misc.imread(map_path) # CAM and convolutional features
new_reshape_size = reshape_size
# Resize map to original size
mapping = resize(mapping, tuple(new_reshape_size), order=1, preserve_range=True)
# Detect regions above a certain percentage of the max heat
bb_thres = np.max(mapping) * percentage_heat
# Compute binary selected region
binary_heat = mapping
binary_heat = np.where(binary_heat > bb_thres, 255, 0)
# Get biggest connected component
min_size = new_reshape_size[0] * new_reshape_size[1] * size_restriction
labeled, _ = ndimage.label(binary_heat) # get connected components
[_, counts] = np.unique(labeled, return_counts=True) # count occurrences
biggest_components = np.argsort(counts[1:])[::-1]
selected_components = [1 if counts[i + 1] >= min_size else 0 for i in
biggest_components] # check minimum size restriction
biggest_components = biggest_components[:min([np.sum(selected_components), 9999])] # get all bboxes
# Extract each component (which will become a bbox prediction)
mapping = mapping / 255.0 # normalize map
# Get bboxes
for selected, comp in zip(selected_components, biggest_components):
if (selected):
max_heat = np.where(labeled == comp + 1, 255, 0) # get the biggest
# Draw bounding box on original image
box = list(bbox(max_heat))
# expand box before final detection
x_exp = box[2] * box_expansion
y_exp = box[3] * box_expansion
box[0] = max([0, box[0] - x_exp / 2])
box[1] = max([0, box[1] - y_exp / 2])
# change width and height by xmax and ymax
box[2] += box[0]
box[3] += box[1]
box[2] = min([new_reshape_size[1] - 1, box[2] + x_exp])
box[3] = min([new_reshape_size[0] - 1, box[3] + y_exp])
predicted_bboxes.append(box)
# Get score for current bbox
score = np.mean(mapping[box[1]:box[3], box[0]:box[2]]) # use mean CAM value of the bbox as a score
predicted_scores.append(score)
# Now apply NMS on all the obtained bboxes
nms_threshold = 0.3
# logging.info('bboxes before NMS: '+str(len(predicted_scores)))
if (len(predicted_scores) > 0):
dets = np.hstack((np.array(predicted_bboxes), np.array(predicted_scores)[:, np.newaxis])).astype(np.float32)
if (use_gpu):
keep = gpu_nms(dets, nms_threshold, device_id=0)
else:
keep = cpu_nms(dets, nms_threshold)
dets = dets[keep, :]
predicted_bboxes = []
predicted_scores = []
for idet in range(dets.shape[0]):
predicted_bboxes.append(dets[idet, :4])
predicted_scores.append(dets[idet, -1])
# logging.info('bboxes after NMS: '+str(len(predicted_scores)))
return [predicted_bboxes, predicted_scores]
def _nms(dets):
return gpu_nms(dets, thresh, device_id)
def getBBoxesFromCAMs(CAMs, reshape_size=[256, 256], percentage_heat=0.4, size_restriction=0.1, box_expansion=0.2,
use_gpu=True):
'''
Reference:
Bolaños, Marc, and Petia Radeva. "Simultaneous Food Localization and Recognition." arXiv preprint arXiv:1604.07953 (2016).
Description:
Extracts a set of bounding boxes from the generated CAMs which contain food instances.
This functions should only be called if the current image has been predicted as Food by the GAP FoodvsNon-food detector!
Arguments:
:param CAMs: list of class activation maps generated by the CAM network
:param reshape_size: reshape proportions used for transorming the CAM for extracting bounding boxes
:param percentage_heat: minimum percentage allowed for considering a detection (aka 't' in reference paper)
:param size_restriction: remove all regions covering less than a certain percentage size of the original image (aka 's' in reference paper)
:param box_expansion: expand the bounding boxes by a certain percentage (aka 'e' in reference paper)
:param use_gpu: boolean indicating if we want to use the GPU for applying NMS
:return: [predicted_bboxes, predicted_scores], containing a list of bboxes coordinates on the first position
and a list of their corresponding scores on the second position
'''
from skimage.transform import resize
from scipy import ndimage
try:
from nms.gpu_nms import gpu_nms
from nms.cpu_nms import cpu_nms
except:
raise Exception(
"Cython is required for running this function:\npip install cython\nRun the following command inside "
"kernel_wrapper/extra/nms after its installation:\npython setup.py build_ext --inplace")
predicted_bboxes = []
predicted_scores = []
# Get all computed maps (if we are also using convolutional features)
all_maps = CAMs
for map in all_maps:
# map = misc.imread(maps_dir[dataset]+'/'+samples_detection[dataset]['all_ids'][s]+'_CAM.jpg') # CAM only
# map = misc.imread(map_path) # CAM and convolutional features
new_reshape_size = reshape_size
# Resize map to original size
map = resize(map, tuple(new_reshape_size), order=1, preserve_range=True)
# Detect regions above a certain percentage of the max heat
bb_thres = np.max(map) * percentage_heat
# Compute binary selected region
binary_heat = map
binary_heat = np.where(binary_heat > bb_thres, 255, 0)
# Get biggest connected component
min_size = new_reshape_size[0] * new_reshape_size[1] * size_restriction
labeled, nr_objects = ndimage.label(binary_heat) # get connected components
[objects, counts] = np.unique(labeled, return_counts=True) # count occurrences
biggest_components = np.argsort(counts[1:])[::-1]
selected_components = [1 if counts[i + 1] >= min_size else 0 for i in
biggest_components] # check minimum size restriction
biggest_components = biggest_components[:min([np.sum(selected_components), 9999])] # get all bboxes
# Extract each component (which will become a bbox prediction)
map = map / 255.0 # normalize map
# Get bboxes
for selected, comp in zip(selected_components, biggest_components):
if (selected):
max_heat = np.where(labeled == comp + 1, 255, 0) # get the biggest
# Draw bounding box on original image
box = list(bbox(max_heat))
# expand box before final detection
x_exp = box[2] * box_expansion
y_exp = box[3] * box_expansion
box[0] = max([0, box[0] - x_exp / 2])
box[1] = max([0, box[1] - y_exp / 2])
# change width and height by xmax and ymax
box[2] += box[0]
box[3] += box[1]
box[2] = min([new_reshape_size[1] - 1, box[2] + x_exp])
box[3] = min([new_reshape_size[0] - 1, box[3] + y_exp])
predicted_bboxes.append(box)
# Get score for current bbox
score = np.mean(map[box[1]:box[3], box[0]:box[2]]) # use mean CAM value of the bbox as a score
predicted_scores.append(score)
# Now apply NMS on all the obtained bboxes
nms_threshold = 0.3
# logging.info('bboxes before NMS: '+str(len(predicted_scores)))
if (len(predicted_scores) > 0):
dets = np.hstack((np.array(predicted_bboxes), np.array(predicted_scores)[:, np.newaxis])).astype(np.float32)
if (use_gpu):
keep = gpu_nms(dets, nms_threshold, device_id=0)
else:
keep = cpu_nms(dets, nms_threshold)
dets = dets[keep, :]
predicted_bboxes = []
predicted_scores = []
for idet in range(dets.shape[0]):
predicted_bboxes.append(dets[idet, :4])
predicted_scores.append(dets[idet, -1])
# logging.info('bboxes after NMS: '+str(len(predicted_scores)))
return [predicted_bboxes, predicted_scores]
def im_detect(net, P, M, rectilinear_mode, cur_im):
if rectilinear_mode:
net_image_size = (1920, 576)
else:
if crop_pano:
net_image_size = (3502, 636)
else:
net_image_size = (3702, 1536)
Pnet = np.copy(P)
image_size = (cur_im.shape[1], cur_im.shape[0])
ratios = (net_image_size[0] / float(image_size[0]),
net_image_size[1] / float(image_size[1]))
Pnet[0, :] *= ratios[0]
Pnet[1, :] *= ratios[1]
show_image = np.copy(cur_im)
cur_im = scipy.misc.imresize(
cur_im, (net_image_size[1], net_image_size[0])).astype(np.float32)
# Concatenate the two images together
input_pair = cur_im
# Normalize the image
input_pair = input_pair - mu
input_pair = np.transpose(input_pair, (2, 0, 1))
input_pair = input_pair[np.newaxis]
forward_kwargs = {'data': input_pair.astype(np.float32, copy=False)}
outputs = net.forward(**forward_kwargs)
proposals_pred = outputs['proposals_score'][:, 1:, 0, 0]
x1 = proposals_pred[:, 0]
y1 = proposals_pred[:, 1]
x2 = proposals_pred[:, 2]
y2 = proposals_pred[:, 3]
proposal_width = x2 - x1
proposal_height = y2 - y1
proposal_ctr_x = x1 + 0.5 * proposal_width
proposal_ctr_y = y1 + 0.5 * proposal_height
score = proposals_pred[:, -1]
num_proposals = score.shape[0]
cls_preds = outputs['cls_pred']
exp_scores = np.exp(cls_preds)
sum_exp_scores = np.sum(exp_scores, 1)
probs = exp_scores / sum_exp_scores[:, np.newaxis]
# Skip the background when computing argmax
det_most_likely_cls_id = np.argmax(probs[:, 1:], 1) + 1
keep_mask = np.logical_and(
np.logical_and(score > proposals_thresh,
np.not_equal(proposal_width, 0)),
np.not_equal(proposal_height, 0))
proposal_width = proposal_width[keep_mask]
proposal_height = proposal_height[keep_mask]
proposal_ctr_x = proposal_ctr_x[keep_mask]
proposal_ctr_y = proposal_ctr_y[keep_mask]
tight_proposal = proposals_pred[keep_mask, :]
quadrant_preds = outputs['quadrant_pred'][keep_mask, :]
select_best_class = False
if select_best_class:
det_cls_id = det_most_likely_cls_id[keep_mask]
keep_mask = np.where(keep_mask)[0] * num_cls + det_cls_id
else:
keep_mask = np.where(keep_mask)[0] * num_cls
det_cls_id = np.tile(cls_ids, keep_mask.size)
keep_mask = np.repeat(keep_mask, len(cls_ids))
keep_mask = keep_mask + det_cls_id
proposal_ctr_x = np.repeat(proposal_ctr_x, len(cls_ids))
proposal_ctr_y = np.repeat(proposal_ctr_y, len(cls_ids))
proposal_width = np.repeat(proposal_width, len(cls_ids))
proposal_height = np.repeat(proposal_height, len(cls_ids))
quadrant_preds = np.repeat(quadrant_preds, len(cls_ids), axis=0)
tight_proposal = np.repeat(tight_proposal, len(cls_ids), axis=0)
probs = probs.reshape(num_proposals * num_cls)[keep_mask]
size_preds = outputs['size_pred'].reshape(num_proposals * num_cls,
-1)[keep_mask, :]
bbox_preds = outputs['bbox_pred'].reshape(num_proposals * num_cls,
-1)[keep_mask, :]
location_preds = outputs['location_pred'].reshape(num_proposals * num_cls,
-1)[keep_mask, :]
orientation_preds = outputs['orientation_pred'].reshape(
num_proposals * num_cls, -1)[keep_mask, :]
distance_pred_a = outputs['distance_pred_a'].reshape(num_proposals *
num_cls)[keep_mask]
distance_pred_b = outputs['distance_pred_b'].reshape(num_proposals *
num_cls)[keep_mask]
all_objects = dict()
targets_size = size_preds
targets_size = targets_size * stat_var_scale + stat_mean_scale
pred_size = targets_size
cur = Frame(rectilinear_mode, image_size, ratios, Pnet, M,
tight_proposal[:, :4], proposal_ctr_x, proposal_ctr_y,
proposal_width, proposal_height, det_cls_id, bbox_preds,
location_preds, orientation_preds, quadrant_preds,
distance_pred_a, distance_pred_b)
cur.pred_size = pred_size
cls_dets = np.hstack((cur.bbox_pred, probs[:, np.newaxis]))
cls_dets[:, 2] += cls_dets[:, 0]
cls_dets[:, 3] += cls_dets[:, 1]
if cls_dets.shape[0] > 0:
nms_keep = gpu_nms(cls_dets.astype(np.float32), nms_thresh)
else:
nms_keep = []
all_objects = dict()
for c in cls_ids:
all_objects[c] = []
for idx in nms_keep:
if det_cls_id[idx] in cls_ids:
all_objects[det_cls_id[idx]].append(
Object(cur, probs[idx], idx, det_cls_id[idx]))
return all_objects
