def __call__(self, bbox_deltas, scores, anchors, im_size, scale=1.):
xp = cuda.get_array_module(bbox_deltas)
bbox_deltas = cuda.to_cpu(bbox_deltas)
scores = cuda.to_cpu(scores)
anchors = cuda.to_cpu(anchors)
height, width = im_size[0], im_size[1]
# bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
# scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
proposals = bbox_transform_inv(anchors, bbox_deltas)
# proposals = clip_boxes(proposals, im_size)
proposals[:, slice(0, 4, 2)] = np.clip(proposals[:, slice(0, 4, 2)], 0,
im_size[0])
proposals[:, slice(1, 4, 2)] = np.clip(proposals[:, slice(1, 4, 2)], 0,
im_size[1])
# Remove predicted boxes with either height or width < threshold
keep = _filter_boxes(proposals, self.min_size * scale)
proposals = proposals[keep, :]
scores = scores[keep]
# Sort (proposal, scores) by score from highest to lowest
# Take top pre_nms_topN
order = scores.ravel().argsort()[::-1]
if self.pre_nms_topN > 0:
order = order[:self.pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
# Apply NMS
# Take after_nms_topN
# Return the top proposals
if xp != np and not self.force_cpu_nms:
keep = non_maximum_suppression(cuda.to_gpu(proposals),
thresh=self.nms_thresh)
keep = cuda.to_cpu(keep)
else:
keep = non_maximum_suppression(proposals, thresh=self.nms_thresh)
if self.post_nms_topN > 0:
keep = keep[:self.post_nms_topN]
proposals = proposals[keep]
# Output ROIs blob
# Batch_size = 1 so all batch_inds are 0
if xp != np:
proposals = cuda.to_gpu(proposals)
return proposals
def find_object_quick(self):
found_objects = []
totalIter = 0
for window in self.window_scores:
totalIter += 1
x = window[0]
y = window[1]
index = window[2]
score = window[3]
if score >= self.detection_threshold:
scale_corr = math.pow(self.w_scaling, index)
x1 = int(x / scale_corr)
y1 = int(y / scale_corr)
x2 = x1 + int(self.w_size[0] / scale_corr)
y2 = y1 + int(self.w_size[1] / scale_corr)
found_objects.append((x1, y1, x2, y2, score))
filtered_bboxes = nms.non_maximum_suppression(found_objects, self.overlap_threshold)
return filtered_bboxes, totalIter
def test_no_remove_box():
boxes, scores = create_non_overlapping_boxes()
nms_boxes, nms_scores = non_maximum_suppression(boxes, scores, 0.55)
# NMS algorithms returns scores and boxes in decrescent order by comparing the score
boxes_scored = sorted(zip(boxes, scores), key=lambda x: x[1], reverse=True)
boxes = [x[0] for x in boxes_scored]
scores = [x[1] for x in boxes_scored]
assert boxes == nms_boxes
assert nms_scores == scores
def _suppress(raw_cls_bbox, raw_prob, n_class, nms_thresh, score_thresh):
# cpu base method
# since the parameter "n_class" is depend on the model's "n_class"
# model's "n_class" is defined by the head part
# it include the background class
bbox = []
label = []
score = []
# skip cls_id = 0 because it is the background class
for l in range(1, n_class):
# (N, 4)
if raw_cls_bbox.size(0) < n_class:
# raw_cls_bbox has less num than n_class, e.g., only 1 bbox but 2 class
cls_bbox_l = raw_cls_bbox[...]
else:
cls_bbox_l = raw_cls_bbox.reshape((-1, n_class, 4))[:, l, :]
# cls_bbox_l = raw_cls_bbox[...]
prob_l = raw_prob[:, l]
mask = prob_l > score_thresh
cls_bbox_l = cls_bbox_l[mask]
prob_l = prob_l[mask]
# NMS return the indices of keep bbox
keep = non_maximum_suppression(cls_bbox_l, prob_l, nms_thresh)
bbox.append(cls_bbox_l[keep])
# The labels are in [0, self.n_class - 2].
label.append(l * torch.ones((keep.nelement(),)))
score.append(prob_l[keep])
# cls_bbox_l = raw_cls_bbox[...]
#
# print(cls_bbox_l.size(), raw_prob.size())
#
# prob_l, _ = torch.max(raw_prob, dim=1)
# mask = prob_l > score_thresh
# cls_bbox_l = cls_bbox_l[mask]
# prob_l = prob_l[mask]
# keep = non_maximum_suppression(cls_bbox_l, prob_l, nms_thresh)
#
# bbox.append(cls_bbox_l[keep])
# # The labels are in [0, self.n_class - 2].
# label.append(torch.argmax(prob_l[keep], dim=1))
# score.append(prob_l[keep])
bbox = torch.cat(bbox, dim=0).int()
label = torch.cat(label, dim=0).int()
score = torch.cat(score, dim=0).float()
return bbox, label, score
def __call__(self,loc,score,anchor,img_size,scale=1.):
'''
Args:
loc:锚点产生的bbox对应到真bbox的偏移和缩放，shape=(R,4)
score:锚点对应的是前景的可能性,shape=(R,)
anchor:锚点产生的bbox的x/y_min/max四个坐标，shape=(R,4)
img_size:图片的原始尺寸，（height，width）
scale:对图片的缩放
return:
一组生成提议框的坐标，（s,4)，数量不超过n_test/train_post_nms
'''
if self.parent_model.training:
n_pre_nms = self.n_train_pre_nms
n_post_nms = self.n_train_post_nms
else:
n_pre_nms = self.n_test_pre_nms
n_post_nms = self.n_test_post_nms
#将生成的锚点框平移缩放成roi
roi = loc2bbox(anchor,loc)
#将超出图片y坐标的上下限重置为0和img_size[0]
roi[:,slice(0,4,2)] = np.clip(roi[:,slice(0,4,2)],0,img_size[0])
#将超出图片x坐标的上下限重置为0和img_size[1]
roi[:,slice(0,4,2)] = np.clip(roi[:,slice(0,4,2)],0,img_size[0])
min_size = self.min_size*scale
hs = roi[:,2] - roi[:,0]
ws = roi[:,3] - roi[:,1]
#shape=(np.array[],),所以通过[0]把数组提取出来，这里是要保留的数据
keep = np.where((hs>=min_size)&(ws>=min_size))[0]
roi = roi[keep,:]
score = score[keep]
#ravel()将函数铺平成一维，argsort()将函数从小到大排序后，返回排序后
#值在原数组里面的序号
order = score.ravel().argsort()[::-1]
if n_pre_nms>0:
order = order[:n_pre_nms]
roi = roi[order,:]
# TODO: apply nms here(eg:threshoid=0.7)
keep = non_maximum_suppression(
cp.ascontiguousarray(cp.asarray(roi)),
thresh=self.nms_thresh)
if n_post_nms>0:
keep=keep[:,n_post_nms]
roi = roi[keep]
return roi
def _suppress(self, raw_cls_bbox, raw_prob):
bbox = list()
label = list()
score = list()
for l in range(1, self.n_class):
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
prob_l = raw_prob[:, l]
mask = prob_l > self.score_thresh
cls_bbox_l = cls_bbox_l[mask]
prob_l = prob_l[mask]
keep = non_maximum_suppression(cls_bbox_l, self.nms_thresh, prob_l)
bbox.append(cls_bbox_l[keep])
label.append((l - 1) * np.ones((len(keep),)))
score.append(prob_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.int32)
score = np.concatenate(score, axis=0).astype(np.float32)
return bbox, label, score
def test_remove_box():
boxes, scores = create_non_overlapping_boxes()
ov_boxes, ov_scores = create_high_overlaping_boxes(boxes)
total_boxes = deepcopy(boxes) + ov_boxes
total_scores = deepcopy(scores) + ov_scores
nms_boxes, nms_scores = non_maximum_suppression(total_boxes, total_scores,
0.55)
print(len(total_boxes))
print(len(boxes))
print(len(nms_boxes))
# NMS algorithms returns scores and boxes in decrescent order by comparing the score
boxes_scored = sorted(zip(boxes, scores), key=lambda x: x[1], reverse=True)
boxes = [x[0] for x in boxes_scored]
scores = [x[1] for x in boxes_scored]
assert boxes == nms_boxes
assert nms_scores == scores
image_list = glob.glob("data/images/*.jpg")
for idx, image_path in enumerate(image_list):
img_name = image_path.split("/")[-1].strip()
print(
f"Processing image {img_name} ({idx+1}/{len(image_list)})")
image_np = cv2.imread(image_path)
boxes, scores, classes = predict(tf_graph, sess, labels_df,
image_np)
nms_classes = []
nms_boxes = []
nms_sores = []
for clss in np.unique(classes):
mask = classes == clss
loc_boxes, loc_sores = non_maximum_suppression(
boxes[mask], scores[mask], NMS_TH)
nms_boxes.extend(loc_boxes)
nms_sores.extend(loc_sores)
nms_classes.extend([clss] * len(loc_boxes))
f, axs = plt.subplots(1, 2)
plot_box_in_image(axs[0], image_np, boxes, scores, classes,
labels_df)
plot_box_in_image(axs[1], image_np, nms_boxes, nms_sores,
nms_classes, labels_df)
axs[0].set_title("Without using NSM")
axs[1].set_title("Using standard NSM")
plt.tight_layout()
plt.savefig(
f"results/{img_name}".replace("jpg", "png"),
format="png",
def find_object(self, image, svm, mirrored_svm):
if self.image_is_equal(image, self.current_image):
return self.find_object_quick()
self.current_image = image.copy()
self.window_scores = []
pyramid = self.create_pyramid(image)
# Calculate amount of windows
hor_amounts = []
ver_amounts = []
for i, img in enumerate(pyramid):
h, w = img.shape[:2]
hor_amounts.append(math.floor((w - self.w_size[0]) / self.w_stride[0]) + 1)
ver_amounts.append(math.floor((h - self.w_size[1]) / self.w_stride[1]) + 1)
totalIter = 0
counter = 0
for i in range(len(pyramid)):
totalIter += hor_amounts[i] * ver_amounts[i]
# Search object in each image of the pyramid
found_objects = []
for index, img in enumerate(pyramid):
hor = hor_amounts[index]
ver = ver_amounts[index]
# Perform sliding window
for i in range(ver):
for j in range(hor):
if counter % 120 == 0:
print("Searching for object...", str(math.ceil(counter / totalIter * 100)) + "%", end='\r')
# Calculate current position of sliding window
x = j * self.w_stride[0]
y = i * self.w_stride[1]
window_hog = self.hog.calc_hog(img, (x, y))
score_normal = svm.classify([window_hog])[0][0]
score_mirrored = mirrored_svm.classify([window_hog])[0][0]
score = max(score_normal, score_mirrored)
self.window_scores.append((x, y, index, score))
if score >= self.detection_threshold:
scale_corr = math.pow(self.w_scaling, index)
x1 = int(x / scale_corr)
y1 = int(y / scale_corr)
x2 = x1 + int(self.w_size[0] / scale_corr)
y2 = y1 + int(self.w_size[1] / scale_corr)
found_objects.append((x1, y1, x2, y2, score))
counter += 1
print("")
filtered_bboxes = nms.non_maximum_suppression(found_objects, self.overlap_threshold)
return filtered_bboxes, totalIter
def __call__(self, loc, score, anchor, img_size, scale=1.):
"""input should be ndarray
Propose RoIs.
Inputs :obj:`loc, score, anchor` refer to the same anchor when indexed
by the same index.
On notations, :math:`R` is the total number of anchors. This is equal
to product of the height and the width of an image and the number of
anchor bases per pixel.
Type of the output is same as the inputs.
Args:
loc (array): Predicted offsets and scaling to anchors.
Its shape is :math:`(R, 4)`.
score (array): Predicted foreground probability for anchors.
Its shape is :math:`(R,)`.
anchor (array): Coordinates of anchors. Its shape is
:math:`(R, 4)`.
img_size (tuple of ints): A tuple :obj:`height, width`,
which contains image size after scaling.
scale (float): The scaling factor used to scale an image after
reading it from a file.
Returns:
array:
An array of coordinates of proposal boxes.
Its shape is :math:`(S, 4)`. :math:`S` is less than
:obj:`self.n_test_post_nms` in test time and less than
:obj:`self.n_train_post_nms` in train time. :math:`S` depends on
the size of the predicted bounding boxes and the number of
bounding boxes discarded by NMS.
"""
# NOTE: when test, remember
# faster_rcnn.eval()
# to set self.traing = False
if self.parent_model.training:
n_pre_nms = self.n_train_pre_nms
n_post_nms = self.n_train_post_nms
else:
n_pre_nms = self.n_test_pre_nms
n_post_nms = self.n_test_post_nms
# Convert anchors into proposal via bbox transformations.
# roi = loc2bbox(anchor, loc)
roi = loc2bbox(anchor, loc)
# Clip predicted boxes to image.
roi[:, slice(0, 4, 2)] = np.clip(roi[:, slice(0, 4, 2)], 0,
img_size[0])
roi[:, slice(1, 4, 2)] = np.clip(roi[:, slice(1, 4, 2)], 0,
img_size[1])
# Remove predicted boxes with either height or width < threshold.
min_size = self.min_size * scale
hs = roi[:, 2] - roi[:, 0]
ws = roi[:, 3] - roi[:, 1]
keep = np.where((hs >= min_size) & (ws >= min_size))[0]
roi = roi[keep, :]
score = score[keep]
# Sort all (proposal, score) pairs by score from highest to lowest.
# Take top pre_nms_topN (e.g. 6000).
order = score.ravel().argsort()[::-1]
if n_pre_nms > 0:
order = order[:n_pre_nms]
roi = roi[order, :]
# Apply nms (e.g. threshold = 0.7).
# Take after_nms_topN (e.g. 300).
# unNOTE: somthing is wrong here!
# TODO: remove cuda.to_gpu
keep = non_maximum_suppression(np.ascontiguousarray(np.asarray(roi)),
thresh=self.nms_thresh)
if n_post_nms > 0:
keep = keep[:n_post_nms]
roi = roi[keep]
return roi