def test_non_maximum_suppression_consistency(self):
bbox = generate_random_bbox(6000, (600, 800), 32, 512)
cpu_selec = non_maximum_suppression(bbox, 0.5)
gpu_selec = non_maximum_suppression(cuda.to_gpu(bbox), 0.5)
np.testing.assert_equal(cpu_selec, cuda.to_cpu(gpu_selec))
def test_non_maximum_suppression_consistency(self):
bbox = generate_random_bbox(6000, (600, 800), 32, 512)
cpu_selec = non_maximum_suppression(bbox, 0.5)
gpu_selec = non_maximum_suppression(cuda.to_gpu(bbox), 0.5)
np.testing.assert_equal(cpu_selec, cuda.to_cpu(gpu_selec))
def check_non_maximum_suppression(self, bbox, threshold, expect):
selec = non_maximum_suppression(bbox, threshold)
self.assertIsInstance(selec, type(bbox))
self.assertEqual(selec.dtype, np.int32)
np.testing.assert_equal(
cuda.to_cpu(selec),
cuda.to_cpu(expect))
def _suppress(raw_bbox, raw_score, nms_thresh, score_thresh):
xp = cuda.get_array_module(raw_bbox, raw_score)
bbox = []
label = []
score = []
for l in range(raw_score.shape[1] - 1):
bbox_l = raw_bbox[:, l + 1]
score_l = raw_score[:, l + 1]
mask = score_l >= score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
order = argsort(-score_l)
bbox_l = bbox_l[order]
score_l = score_l[order]
indices = utils.non_maximum_suppression(bbox_l, nms_thresh)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(bbox_l)
label.append(xp.array((l, ) * len(bbox_l)))
score.append(score_l)
bbox = xp.vstack(bbox).astype(np.float32)
label = xp.hstack(label).astype(np.int32)
score = xp.hstack(score).astype(np.float32)
return bbox, label, score
def _suppress(self, raw_cls_bbox, raw_prob):
bbox = list()
label = list()
score = list()
# skip cls_id = 0 because it is the background class
for l in range(1, self.n_class):
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
prob_l = raw_prob[:, l]
# thresholding by score
keep = prob_l > self.score_thresh
cls_bbox_l = cls_bbox_l[keep]
prob_l = prob_l[keep]
# thresholding by nms
keep = non_maximum_suppression(
cls_bbox_l, self.nms_thresh, prob_l)
bbox.append(cls_bbox_l[keep])
# The labels are in [0, self.n_class - 2].
label.append((l - 1) * np.ones((len(keep),)))
score.append(prob_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.int32)
score = np.concatenate(score, axis=0).astype(np.float32)
return bbox, label, score
def _suppress(self, raw_bbox, raw_score):
xp = self.xp
bbox = list()
label = list()
score = list()
for l in range(self.n_fg_class):
bbox_l = raw_bbox
# the l-th class corresponds for the (l + 1)-th column.
score_l = raw_score[:, l + 1]
mask = score_l >= self.score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
if self.nms_thresh is not None:
indices = utils.non_maximum_suppression(
bbox_l, self.nms_thresh, score_l)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(bbox_l)
label.append(xp.array((l,) * len(bbox_l)))
score.append(score_l)
bbox = xp.vstack(bbox).astype(np.float32)
label = xp.hstack(label).astype(np.int32)
score = xp.hstack(score).astype(np.float32)
return bbox, label, score
def _suppress(self, raw_cls_bbox, raw_cls_roi, raw_prob, raw_mask):
bbox = list()
roi = list()
label = list()
score = list()
mask = list()
for l in range(1, self.n_class):
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
cls_roi_l = raw_cls_roi.reshape((-1, self.n_class, 4))[:, l, :]
prob_l = raw_prob[:, l]
lmask = prob_l > self.score_thresh
cls_bbox_l = cls_bbox_l[lmask]
cls_roi_l = cls_roi_l[lmask]
prob_l = prob_l[lmask]
mask_l = raw_mask[:, l]
mask_l = mask_l[lmask]
keep = non_maximum_suppression(cls_bbox_l, self.nms_thresh, prob_l)
bbox.append(cls_bbox_l[keep])
roi.append(cls_roi_l[keep])
#labels are in [0, self.nclass - 2].
label.append((l - 1) * np.ones((len(keep), )))
score.append(prob_l[keep])
mask.append(mask_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
roi = np.concatenate(roi, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.float32)
score = np.concatenate(score, axis=0).astype(np.float32)
mask = np.concatenate(mask, axis=0).astype(np.float32)
return bbox, roi, label, score, mask
def _suppress(self, raw_cls_bbox, raw_prob, raw_roi, raw_level):
bbox = []
label = []
score = []
roi = []
level = []
# skip cls_id = 0 because it is the background class
# -> maskは0から始まるから、l-1を使う
# -> あーしまったTrainChainで最後のクラスToothBlushは範囲外になっておるわ・・
for l in range(1, self.n_class):
if self.predict_mask and l == self.n_class - 1:
# まったく本質的でないのだか、maskを推定するときの学習でオフセットを間違えており、
# l == self.n_class-1でindex out of boundsする？要検証
continue
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
prob_l = raw_prob[:, l]
mask = prob_l > self.score_thresh
cls_bbox_l = cls_bbox_l[mask]
prob_l = prob_l[mask]
keep = non_maximum_suppression(cls_bbox_l, self.nms_thresh, prob_l)
bbox.append(cls_bbox_l[keep])
# The labels are in [0, self.n_class - 2].
label.append((l - 1) * np.ones((len(keep), )))
score.append(prob_l[keep])
raw_roi_l = raw_roi[:, l, :][mask]
roi.append(raw_roi_l[keep])
level_l = raw_level[mask]
level.append(level_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.int32)
score = np.concatenate(score, axis=0).astype(np.float32)
roi = np.concatenate(roi, axis=0)
level = np.concatenate(level, axis=0).astype(np.int32)
return bbox, label, score, roi, level
def check_non_maximum_suppression_options(
self, bbox, threshold, score, limit):
# Pass all options to the tested function
scored_selec = non_maximum_suppression(bbox, threshold, score, limit)
self.assertIsInstance(scored_selec, type(bbox))
# Reorder inputs befor passing it to the function.
# Reorder the outputs according to scores.
order = score.argsort()[::-1]
reordered_selec = non_maximum_suppression(
bbox[order], threshold, score=None, limit=None)
reordered_selec = reordered_selec[:limit]
reordered_selec = order[reordered_selec]
np.testing.assert_equal(
cuda.to_cpu(scored_selec), cuda.to_cpu(reordered_selec))
def decode(self, mb_loc, mb_conf, nms_thresh, score_thresh):
xp = self.xp
mb_bbox = self._default_bbox.copy()
mb_bbox[:, :2] += mb_loc[:, :2] * self._variance[
0] * self._default_bbox[:, 2:]
mb_bbox[:, 2:] *= xp.exp(mb_loc[:, 2:] * self._variance[1])
mb_bbox[:, :2] -= mb_bbox[:, 2:] / 2
mb_bbox[:, 2:] += mb_bbox[:, :2]
if xp == np:
mb_conf[mb_conf > 88.72] = 88.72 # avoid overflow
mb_score = xp.exp(mb_conf)
mb_score /= mb_score.sum(axis=1, keepdims=True)
# intra-class non-maximum suppression
bbox = []
label = []
score = []
for l in range(mb_conf.shape[1] - 1):
bbox_l = mb_bbox
score_l = mb_score[:, l + 1]
mask = score_l >= score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
indices = utils.non_maximum_suppression(bbox_l, nms_thresh,
score_l)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(bbox_l)
label.append(xp.array((l, ) * len(bbox_l)))
score.append(score_l)
# inter-class non-maximum suppression
bbox = xp.vstack(bbox)
label = xp.hstack(label)
score = xp.hstack(score)
indices = utils.non_maximum_suppression(bbox, nms_thresh, score)
bbox = bbox[indices].astype(np.float32)
label = label[indices].astype(np.int32)
score = score[indices].astype(np.float32)
return bbox, label, score
def check_non_maximum_suppression_options(self, bbox, threshold, score,
limit):
# Pass all options to the tested function
scored_selec = non_maximum_suppression(bbox, threshold, score, limit)
self.assertIsInstance(scored_selec, type(bbox))
# Reorder inputs befor passing it to the function.
# Reorder the outputs according to scores.
order = score.argsort()[::-1]
reordered_selec = non_maximum_suppression(bbox[order],
threshold,
score=None,
limit=None)
reordered_selec = reordered_selec[:limit]
reordered_selec = order[reordered_selec]
np.testing.assert_equal(cuda.to_cpu(scored_selec),
cuda.to_cpu(reordered_selec))
def get_humans_by_feature(model,
feature_map,
detection_thresh=0.15,
min_num_keypoints=-1):
resp, conf, x, y, w, h, e = feature_map
start = time.time()
delta = resp * conf
K = len(model.keypoint_names)
outW, outH = model.outsize
ROOT_NODE = 0 # instance
start = time.time()
rx, ry = model.restore_xy(x, y)
rw, rh = model.restore_size(w, h)
ymin, ymax = ry - rh / 2, ry + rh / 2
xmin, xmax = rx - rw / 2, rx + rw / 2
bbox = np.array([ymin, xmin, ymax, xmax])
bbox = bbox.transpose(1, 2, 3, 0)
root_bbox = bbox[ROOT_NODE]
score = delta[ROOT_NODE]
candidate = np.where(score > detection_thresh)
score = score[candidate]
root_bbox = root_bbox[candidate]
selected = non_maximum_suppression(bbox=root_bbox, thresh=0.3, score=score)
root_bbox = root_bbox[selected]
#logger.info('detect instance {:.5f}'.format(time.time() - start))
start = time.time()
humans = []
e = e.transpose(0, 3, 4, 1, 2)
ei = 0 # index of edges which contains ROOT_NODE as begin
# alchemy_on_humans
for hxw in zip(candidate[0][selected], candidate[1][selected]):
human = {ROOT_NODE: bbox[(ROOT_NODE, hxw[0], hxw[1])]} # initial
for graph in DIRECTED_GRAPHS:
eis, ts = graph
i_h, i_w = hxw
for ei, t in zip(eis, ts):
index = (ei, i_h, i_w) # must be tuple
u_ind = np.unravel_index(np.argmax(e[index]), e[index].shape)
j_h = i_h + u_ind[0] - model.local_grid_size[1] // 2
j_w = i_w + u_ind[1] - model.local_grid_size[0] // 2
if j_h < 0 or j_w < 0 or j_h >= outH or j_w >= outW:
break
if delta[t, j_h, j_w] < detection_thresh:
break
human[t] = bbox[(t, j_h, j_w)]
i_h, i_w = j_h, j_w
if min_num_keypoints <= len(human) - 1:
humans.append(human)
#logger.info('alchemy time {:.5f}'.format(time.time() - start))
logger.info('num humans = {}'.format(len(humans)))
return humans
def mask_voting(
rois, cls_probs, mask_probs,
n_class, H, W,
score_thresh=0.7,
nms_thresh=0.3,
mask_merge_thresh=0.5,
binary_thresh=0.4):
mask_size = mask_probs.shape[-1]
v_labels = np.empty((0, ), dtype=np.int32)
v_masks = np.empty((0, mask_size, mask_size), dtype=np.float32)
v_bboxes = np.empty((0, 4), dtype=np.float32)
v_cls_probs = np.empty((0, ), dtype=np.float32)
for l in range(0, n_class - 1):
# non maximum suppression
cls_prob_l = cls_probs[:, l+1]
thresh_mask = cls_prob_l >= 0.001
bbox_l = rois[thresh_mask]
cls_prob_l = cls_prob_l[thresh_mask]
keep = non_maximum_suppression(
bbox_l, nms_thresh, cls_prob_l, limit=100)
bbox_l = bbox_l[keep]
cls_prob_l = cls_prob_l[keep]
n_bbox_l = len(bbox_l)
v_mask_l = np.zeros((n_bbox_l, mask_size, mask_size))
v_bbox_l = np.zeros((n_bbox_l, 4))
for i, bbox in enumerate(bbox_l):
iou = bbox_iou(rois, bbox[np.newaxis, :])
idx = np.where(iou > mask_merge_thresh)[0]
mask_weights = cls_probs[idx, l + 1]
mask_weights = mask_weights / mask_weights.sum()
mask_prob_l = mask_probs[idx]
rois_l = rois[idx]
orig_mask, v_bbox_l[i] = mask_aggregation(
rois_l, mask_prob_l, mask_weights, H, W, binary_thresh)
v_mask_l[i] = cv2.resize(
orig_mask.astype(np.float32), (mask_size, mask_size))
score_thresh_mask = cls_prob_l > score_thresh
v_cls_prob_l = cls_prob_l[score_thresh_mask]
v_mask_l = v_mask_l[score_thresh_mask]
v_bbox_l = v_bbox_l[score_thresh_mask]
v_label_l = np.repeat(l, v_bbox_l.shape[0])
v_cls_probs = np.concatenate((v_cls_probs, v_cls_prob_l))
v_masks = np.concatenate((v_masks, v_mask_l))
v_bboxes = np.concatenate((v_bboxes, v_bbox_l))
v_labels = np.concatenate((v_labels, v_label_l))
return v_labels, v_masks, v_bboxes, v_cls_probs
def detect(
self,
image: Image.Image,
nms_iou_threshold: float = 0.5
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Detect characters from the image."""
img_w, img_h = image.size
if img_w < img_h:
w = self.image_min_side
h = img_h * self.image_min_side / img_w
h = 32 * int(round(h / 32))
else:
h = self.image_min_side
w = img_w * self.image_min_side / img_h
w = 32 * int(round(w / 32))
image = image.resize((w, h), resample=Image.BILINEAR)
img = np.asarray(image, dtype=np.float32).transpose(2, 0, 1)
imgs = img.reshape(1, *img.shape)
if self.xp != np:
imgs = cuda.to_gpu(imgs)
imgs = (imgs - 127.5) / 128.0
with chainer.using_config('train', False), chainer.no_backprop_mode():
heatmap = self(imgs)
heatmap = heatmap.array
heatmap[:-4] = _sigmoid(heatmap[:-4])
bboxes, _, scores = heatmap_to_labeled_bboxes(heatmap,
self.score_threshold)
bboxes, scores = bboxes[0], scores[0]
hm_h, hm_w = heatmap.shape[2:4]
bboxes[:, 0::2] *= img_w / hm_w
bboxes[:, 1::2] *= img_h / hm_h
keep = non_maximum_suppression(bboxes, nms_iou_threshold, score=scores)
bboxes = bboxes[keep]
scores = scores[keep]
if self.xp != np:
bboxes = cuda.to_cpu(bboxes)
scores = cuda.to_cpu(scores)
return bboxes, scores
def _filter_overlapping_bboxs(self, mb_boxs, mb_confs):
confs = []
labels = []
for box, conf, label in zip(mb_boxs, mb_confs, self.gt_mb_labels):
indices = utils.non_maximum_suppression(box, self.nms_thresh)
confs.append(conf[indices])
if chainer.cuda.available:
labels.append(label[indices].get())
else:
labels.append(label[indices])
confs = F.concat(confs, axis=0)
labels = np.concatenate(labels)
return zip(labels, confs)
def filter_overlapping_bboxs(self, mb_boxs, mb_confs, gt_labels):
confs = []
labels = []
for box, conf, label in zip(mb_boxs, mb_confs, gt_labels):
indices = non_maximum_suppression(box, 0.5)
# Add more beautiful version of this nms-thresh
confs.append(conf[indices])
if chainer.cuda.available:
labels.append(label[indices.get()])
else:
labels.append(label[indices])
confs = F.concat(confs, axis=0)
labels = np.concatenate(labels)
return zip(labels, confs)
def _decode(self, loc, obj, conf):
raw_bbox = self._default_bbox.copy()
raw_bbox[:, :2] += 1 / (1 + self.xp.exp(-loc[:, :2]))
raw_bbox[:, :2] *= self._step[:, None]
raw_bbox[:, 2:] *= self.xp.exp(loc[:, 2:])
raw_bbox[:, :2] -= raw_bbox[:, 2:] / 2
raw_bbox[:, 2:] += raw_bbox[:, :2]
obj = 1 / (1 + self.xp.exp(-obj))
conf = 1 / (1 + self.xp.exp(-conf))
raw_score = obj[:, None] * conf
np_step = cuda.to_cpu(self._step)
step_list = sorted(set(np_step), key=np_step.tolist().index)
bbox = []
label = []
score = []
layer_id = []
for l in range(self.n_fg_class):
bbox_l = raw_bbox
score_l = raw_score[:, l]
mask = score_l >= self.score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
step_l = self._step[mask]
layer_id_l = np.array([step_list.index(step) for step in step_l])
indices = utils.non_maximum_suppression(
bbox_l, self.nms_thresh, score_l)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
layer_id_l = layer_id_l[cuda.to_cpu(indices)]
bbox.append(bbox_l)
label.append(self.xp.array((l,) * len(bbox_l)))
score.append(score_l)
layer_id.append(layer_id_l)
bbox = self.xp.vstack(bbox).astype(np.float32)
label = self.xp.hstack(label).astype(np.int32)
score = self.xp.hstack(score).astype(np.float32)
layer_id = self.xp.hstack(layer_id).astype(np.int32)
return bbox, label, score, layer_id
def _suppress_each_box(self, raw_cls_bbox, raw_prob):
xp = np #model.xp
bbox = []
label = []
prob = []
best_class = raw_prob.argmax(axis=1)
self.raw_cls_bbox = raw_cls_bbox
if True:
best_class = best_class[:len(raw_cls_bbox)]
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))
cls_bbox_l = xp.array(
[cls_bbox_l[i, item, :] for i, item in enumerate(best_class)])
non_bg_mask = best_class > 0
#prob_l = raw_prob[:, best_class]
prob_l = xp.array(
[raw_prob[i, item] for i, item in enumerate(best_class)])
self.prob_l = prob_l
mask = np.logical_and(prob_l > self.score_thresh, best_class > 0)
cls_bbox_l = cls_bbox_l[mask]
prob_l = prob_l[mask]
#best_class_mask = best_class[mask]
keep = non_maximum_suppression(cls_bbox_l, self.nms_thresh, prob_l)
bbox.append(cls_bbox_l[keep])
# The labels are in [0, self.n_class - 2].
#label.append((l - 1) * np.ones((len(keep),)))
self.out = prob_l
self.best_class = best_class
self.mask = mask
self.keep = keep
self.raw_prob = raw_prob
label.append(best_class[mask][keep] - 1)
prob.append(prob_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.int32)
prob = np.concatenate(prob, axis=0).astype(np.float32)
return bbox, label, prob
def detect(self, image: Image.Image):
# get all prediction results
pred_bboxes_set = []
pred_scores_set = []
for detector in self.detectors:
bboxes, scores = detector.detect(image)
pred_bboxes_set.append(bboxes)
pred_scores_set.append(scores)
all_bboxes = np.concatenate(pred_bboxes_set)
all_scores = np.concatenate(pred_scores_set)
# apply NMS to obtain base bounding boxes for refinement
keep = non_maximum_suppression(all_bboxes,
thresh=0.3,
score=all_scores)
base_bboxes = all_bboxes[keep]
base_scores = all_scores[keep]
# get matched bboxes
iou_mat = calc_iou_mat(base_bboxes, all_bboxes)
match_mat = iou_mat >= 0.5
# refine bboxes by bbox voting
refined_bboxes = np.empty_like(base_bboxes)
refined_scores = np.empty_like(base_scores)
votes = np.sum(match_mat, axis=1)
for i in range(len(base_bboxes)):
match = match_mat[i]
scores = all_scores[match]
bboxes = all_bboxes[match]
refined_bboxes[i] = np.sum(scores[:, None] * bboxes,
axis=0) / scores.sum()
refined_scores[i] = np.average(scores)
min_votes_mask = votes >= self.min_votes
return refined_bboxes[min_votes_mask], refined_scores[min_votes_mask]
def _suppress(self, raw_cls_bbox, raw_prob):
bbox = []
label = []
score = []
# skip cls_id = 0 because it is the background class
for l in range(1, self.n_class):
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
prob_l = raw_prob[:, l]
mask = prob_l > self.score_thresh
cls_bbox_l = cls_bbox_l[mask]
prob_l = prob_l[mask]
keep = non_maximum_suppression(
cls_bbox_l, self.nms_thresh, prob_l)
bbox.append(cls_bbox_l[keep])
# The labels are in [0, self.n_class - 2].
label.append((l - 1) * np.ones((len(keep),)))
score.append(prob_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.int32)
score = np.concatenate(score, axis=0).astype(np.float32)
return bbox, label, score
def _decode(self, loc, obj, conf):
raw_bbox = self._default_bbox.copy()
raw_bbox[:, :2] += 1 / (1 + self.xp.exp(-loc[:, :2]))
raw_bbox[:, 2:] *= self.xp.exp(loc[:, 2:])
raw_bbox[:, :2] -= raw_bbox[:, 2:] / 2
raw_bbox[:, 2:] += raw_bbox[:, :2]
raw_bbox *= self.insize / self.extractor.grid
obj = 1 / (1 + self.xp.exp(-obj))
conf = self.xp.exp(conf)
conf /= conf.sum(axis=1, keepdims=True)
raw_score = obj[:, None] * conf
bbox = []
label = []
score = []
for l in range(self.n_fg_class):
bbox_l = raw_bbox
score_l = raw_score[:, l]
mask = score_l >= self.score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
indices = utils.non_maximum_suppression(bbox_l, self.nms_thresh,
score_l)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(bbox_l)
label.append(self.xp.array((l, ) * len(bbox_l)))
score.append(score_l)
bbox = self.xp.vstack(bbox).astype(np.float32)
label = self.xp.hstack(label).astype(np.int32)
score = self.xp.hstack(score).astype(np.float32)
return bbox, label, score
def _decode(self, loc, conf):
raw_bbox = self._default_bbox.copy()
raw_bbox[:, :2] += 1 / (1 + self.xp.exp(-loc[:, :2]))
raw_bbox[:, :2] *= self._step[:, None]
raw_bbox[:, 2:] *= self.xp.exp(loc[:, 2:])
raw_bbox[:, :2] -= raw_bbox[:, 2:] / 2
raw_bbox[:, 2:] += raw_bbox[:, :2]
conf = 1 / (1 + self.xp.exp(-conf))
raw_score = conf[:, 0, None] * conf[:, 1:]
bbox = []
label = []
score = []
for l in range(self.n_fg_class):
bbox_l = raw_bbox
score_l = raw_score[:, l]
mask = score_l >= self.score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
indices = utils.non_maximum_suppression(
bbox_l, self.nms_thresh, score_l)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(cuda.to_cpu(bbox_l))
label.append(np.array((l,) * len(bbox_l)))
score.append(cuda.to_cpu(score_l))
bbox = np.vstack(bbox).astype(np.float32)
label = np.hstack(label).astype(np.int32)
score = np.hstack(score).astype(np.float32)
return bbox, label, score
def decode(self, mb_loc, mb_conf, nms_thresh=0.45, score_thresh=0.6):
"""Decodes back to coordinates and classes of bounding boxes.
This method decodes :obj:`mb_loc` and :obj:`mb_conf` returned
by a SSD network back to :obj:`bbox`, :obj:`label` and :obj:`score`.
Args:
mb_loc (array): A float array whose shape is
:math:`(K, 4)`, :math:`K` is the number of
default bounding boxes.
mb_conf (array): A float array whose shape is
:math:`(K, n\_fg\_class + 1)`.
nms_thresh (float): The threshold value
for :func:`~chainercv.utils.non_maximum_suppression`.
The default value is :obj:`0.45`.
score_thresh (float): The threshold value for confidence score.
If a bounding box whose confidence score is lower than
this value, the bounding box will be suppressed.
The default value is :obj:`0.6`.
Returns:
tuple of three arrays:
This method returns a tuple of three arrays,
:obj:`(bbox, label, score)`.
* **bbox**: A float array of shape :math:`(R, 4)`, \
where :math:`R` is the number of bounding boxes in a image. \
Each bouding box is organized by \
:math:`(y_{min}, x_{min}, y_{max}, x_{max})` \
in the second axis.
* **label** : An integer array of shape :math:`(R,)`. \
Each value indicates the class of the bounding box.
* **score** : A float array of shape :math:`(R,)`. \
Each value indicates how confident the prediction is.
"""
xp = self.xp
# (center_y, center_x, height, width)
mb_bbox = self._default_bbox.copy()
mb_bbox[:, :2] += mb_loc[:, :2] * self._variance[0] \
* self._default_bbox[:, 2:]
mb_bbox[:, 2:] *= xp.exp(mb_loc[:, 2:] * self._variance[1])
# (center_y, center_x, height, width) -> (y_min, x_min, height, width)
mb_bbox[:, :2] -= mb_bbox[:, 2:] / 2
# (center_y, center_x, height, width) -> (y_min, x_min, y_max, x_max)
mb_bbox[:, 2:] += mb_bbox[:, :2]
# softmax
mb_score = xp.exp(mb_conf)
mb_score /= mb_score.sum(axis=1, keepdims=True)
bbox = list()
label = list()
score = list()
for l in range(mb_conf.shape[1] - 1):
bbox_l = mb_bbox
# the l-th class corresponds for the (l + 1)-th column.
score_l = mb_score[:, l + 1]
mask = score_l >= score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
if nms_thresh is not None:
indices = utils.non_maximum_suppression(
bbox_l, nms_thresh, score_l)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(bbox_l)
label.append(xp.array((l, ) * len(bbox_l)))
score.append(score_l)
bbox = xp.vstack(bbox).astype(np.float32)
label = xp.hstack(label).astype(np.int32)
score = xp.hstack(score).astype(np.float32)
return bbox, label, score
def __call__(self, bbox, score):
selc = non_maximum_suppression(bbox, self._thresh, score)
return selc
def check_non_maximum_suppression_zero_legnth_bbox(
self, bbox, threshold):
selec = non_maximum_suppression(bbox, threshold)
self.assertIsInstance(selec, type(bbox))
self.assertEqual(selec.shape, (0,))
def mask_voting(roi_cmask_prob,
bbox,
roi_cls_prob,
size,
score_thresh,
nms_thresh,
mask_merge_thresh,
binary_thresh,
limit=100,
bg_label=0):
"""Refine mask probabilities by merging multiple masks.
First, this function discard invalid masks with non maximum suppression.
Then, it merges masks with weight calculated from class probabilities and
iou.
This function improves the mask qualities by merging overlapped masks
predicted as the same object class.
Here are notations used.
* :math:`R'` is the total number of RoIs produced across batches.
* :math:`L` is the number of classes excluding the background.
* :math:`RH` is the height of pooled image.
* :math:`RW` is the height of pooled image.
Args:
roi_cmask_prob (array): A mask probability array whose shape is
:math:`(R, RH, RW)`.
bbox (array): A bounding box array whose shape is
:math:`(R, 4)`.
cls_prob (array): A class probability array whose shape is
:math:`(R, L + 1)`.
size (tuple of int): Original image size.
score_thresh (float): A threshold value of the class score.
nms_thresh (float): A threshold value of non maximum suppression.
mask_merge_thresh (float): A threshold value of the bounding box iou
for mask merging.
binary_thresh (float): A threshold value of mask score
for mask merging.
limit (int): The maximum number of outputs.
bg_label (int): The id of the background label.
Returns:
array, array, array, array:
* **v_cmask_prob**: Merged mask probability. Its shapes is \
:math:`(N, RH, RW)`.
* **v_bbox**: Bounding boxes for the merged masks. Its shape is \
:math:`(N, 4)`.
* **v_label**: Class labels for the merged masks. Its shape is \
:math:`(N, )`.
* **v_score**: Class probabilities for the merged masks. Its shape \
is :math:`(N, )`.
"""
roi_cmask_size = roi_cmask_prob.shape[1:]
n_class = roi_cls_prob.shape[1]
v_cmask_prob = []
v_bbox = []
v_label = []
v_cls_prob = []
cls_score = []
cls_bbox = []
for label in range(0, n_class):
# background
if label == bg_label:
continue
# non maximum suppression
score_l = roi_cls_prob[:, label]
keep_indices = non_maximum_suppression(bbox, nms_thresh, score_l)
bbox_l = bbox[keep_indices]
score_l = score_l[keep_indices]
cls_bbox.append(bbox_l)
cls_score.append(score_l)
sorted_score = np.sort(np.concatenate(cls_score))[::-1]
n_keep = min(len(sorted_score), limit)
score_thresh = max(sorted_score[n_keep - 1], score_thresh)
for label in range(0, n_class):
# background
if label == bg_label:
continue
bbox_l = cls_bbox[label - 1]
score_l = cls_score[label - 1]
keep_indices = np.where(score_l >= score_thresh)
bbox_l = bbox_l[keep_indices]
score_l = score_l[keep_indices]
v_cmask_prob_l = []
v_bbox_l = []
v_score_l = []
for i, bb in enumerate(bbox_l):
iou = bbox_iou(bbox, bb[np.newaxis, :])
keep_indices = np.where(iou >= mask_merge_thresh)[0]
cmask_weight = roi_cls_prob[keep_indices, label]
cmask_weight = cmask_weight / cmask_weight.sum()
cmask_prob_i = roi_cmask_prob[keep_indices]
bbox_i = bbox[keep_indices]
m_cmask, m_bbox = _mask_aggregation(bbox_i, cmask_prob_i,
cmask_weight, size,
binary_thresh)
if m_cmask is not None and m_bbox is not None:
m_cmask = resize(m_cmask.astype(np.float32), roi_cmask_size)
v_cmask_prob_l.append(m_cmask)
v_bbox_l.append(m_bbox)
v_score_l.append(score_l[i])
if len(v_cmask_prob_l) > 0:
v_cmask_prob_l = np.concatenate(v_cmask_prob_l)
v_bbox_l = np.concatenate(v_bbox_l)
v_score_l = np.array(v_score_l)
v_label_l = np.repeat(label - 1, v_bbox_l.shape[0])
v_label_l = v_label_l.astype(np.int32)
v_cmask_prob.append(v_cmask_prob_l)
v_bbox.append(v_bbox_l)
v_label.append(v_label_l)
v_cls_prob.append(v_score_l)
if len(v_cmask_prob) > 0:
v_cmask_prob = np.concatenate(v_cmask_prob)
v_bbox = np.concatenate(v_bbox)
v_label = np.concatenate(v_label)
v_cls_prob = np.concatenate(v_cls_prob)
else:
v_cmask_prob = np.empty((0, roi_cmask_size[0], roi_cmask_size[1]))
v_bbox = np.empty((0, 4))
v_label = np.empty((0, ))
v_cls_prob = np.empty((0, ))
return v_cmask_prob, v_bbox, v_label, v_cls_prob
def decode(self,
arm_loc,
arm_conf,
odm_loc,
odm_conf,
nms_thresh=0.45,
score_thresh=0.6):
xp = self.xp
# (center_y, center_x, height, width)
mb_bbox = self._default_bbox.copy()
mb_bbox[:, :2] += arm_loc[:, :2] * self._variance[0] \
* self._default_bbox[:, 2:]
mb_bbox[:, 2:] *= xp.exp(arm_loc[:, 2:] * self._variance[1])
# Anchor refinement
mb_bbox[:, :2] += odm_loc[:, :2] * self._variance[0] \
* mb_bbox[:, 2:]
mb_bbox[:, 2:] *= xp.exp(odm_loc[:, 2:] * self._variance[1])
# (center_y, center_x, height, width) -> (y_min, x_min, height, width)
mb_bbox[:, :2] -= mb_bbox[:, 2:] / 2
# (center_y, center_x, height, width) -> (y_min, x_min, y_max, x_max)
mb_bbox[:, 2:] += mb_bbox[:, :2]
# softmax
mb_score = xp.exp(odm_conf)
mb_score /= mb_score.sum(axis=1, keepdims=True)
objectness = xp.exp(arm_conf)
negativeness = xp.exp(1 - arm_conf)
objectness /= objectness + negativeness
# negative anchor filtering
mb_score[objectness <= 0.01] = 0
bbox = list()
label = list()
score = list()
for l in range(odm_conf.shape[1] - 1):
bbox_l = mb_bbox
# the l-th class corresponds for the (l + 1)-th column.
score_l = mb_score[:, l + 1]
mask = score_l >= score_thresh
bbox_l = bbox_l[mask]
score_l = score_l[mask]
if nms_thresh is not None:
indices = utils.non_maximum_suppression(bbox_l,
nms_thresh,
score_l,
limit=400)
bbox_l = bbox_l[indices]
score_l = score_l[indices]
bbox.append(bbox_l)
label.append(xp.array((l, ) * len(bbox_l)))
score.append(score_l)
bbox = xp.vstack(bbox).astype(np.float32)
label = xp.hstack(label).astype(np.int32)
score = xp.hstack(score).astype(np.float32)
return bbox, label, score
def check_non_maximum_suppression_zero_legnth_bbox(self, bbox, threshold):
selec = non_maximum_suppression(bbox, threshold)
self.assertIsInstance(selec, type(bbox))
self.assertEqual(selec.shape, (0, ))
def decode(self, locs, confs, anchors, in_shape):
if chainer.config.train:
nms_limit_pre = self._train_nms_limit_pre
nms_limit_post = self._train_nms_limit_post
else:
nms_limit_pre = self._test_nms_limit_pre
nms_limit_post = self._test_nms_limit_post
rois = []
roi_indices = []
for i in range(in_shape[0]):
roi = []
conf = []
for l in range(len(self._scales)):
loc_l = locs[l].array[i]
conf_l = confs[l].array[i]
roi_l = anchors[l].copy()
# tlbr -> yxhw
roi_l[:, 2:] -= roi_l[:, :2]
roi_l[:, :2] += roi_l[:, 2:] / 2
# offset
roi_l[:, :2] += loc_l[:, :2] * roi_l[:, 2:]
roi_l[:, 2:] *= self.xp.exp(
self.xp.minimum(loc_l[:, 2:], exp_clip))
# yxhw -> tlbr
roi_l[:, :2] -= roi_l[:, 2:] / 2
roi_l[:, 2:] += roi_l[:, :2]
# clip
roi_l[:, :2] = self.xp.maximum(roi_l[:, :2], 0)
roi_l[:, 2:] = self.xp.minimum(roi_l[:, 2:],
self.xp.array(in_shape[2:]))
order = _argsort(-conf_l)[:nms_limit_pre]
roi_l = roi_l[order]
conf_l = conf_l[order]
mask = (roi_l[:, 2:] - roi_l[:, :2] > 0).all(axis=1)
roi_l = roi_l[mask]
conf_l = conf_l[mask]
indices = utils.non_maximum_suppression(roi_l,
self._nms_thresh,
limit=nms_limit_post)
roi_l = roi_l[indices]
conf_l = conf_l[indices]
roi.append(roi_l)
conf.append(conf_l)
roi = self.xp.vstack(roi).astype(np.float32)
conf = self.xp.hstack(conf).astype(np.float32)
order = _argsort(-conf)[:nms_limit_post]
roi = roi[order]
rois.append(roi)
roi_indices.append(self.xp.array((i, ) * len(roi)))
rois = self.xp.vstack(rois).astype(np.float32)
roi_indices = self.xp.hstack(roi_indices).astype(np.int32)
return rois, roi_indices
def check_non_maximum_suppression(self, bbox, threshold, expect):
selec = non_maximum_suppression(bbox, threshold)
self.assertIsInstance(selec, type(bbox))
self.assertEqual(selec.dtype, np.int32)
np.testing.assert_equal(cuda.to_cpu(selec), cuda.to_cpu(expect))
def mask_voting(
rois, mask_probs, cls_probs,
n_class, H, W,
score_thresh=0.7,
nms_thresh=0.3,
mask_merge_thresh=0.5,
binary_thresh=0.4, max_num=100):
mask_size = mask_probs.shape[-1]
v_labels = np.empty((0, ), dtype=np.int32)
v_masks = np.empty((0, mask_size, mask_size), dtype=np.float32)
v_bboxes = np.empty((0, 4), dtype=np.float32)
v_cls_probs = np.empty((0, ), dtype=np.float32)
tmp_all_scores = np.empty((0, ), dtype=np.float32)
tmp_cls_probs = []
tmp_bbox = []
for label in range(0, n_class):
if label == 0:
# l == 0 is background
continue
# non maximum suppression
cls_prob_l = cls_probs[:, label]
keep_indices = non_maximum_suppression(
rois, nms_thresh, cls_prob_l, limit=max_num)
bbox_l = rois[keep_indices]
cls_prob_l = cls_prob_l[keep_indices]
tmp_bbox.append(bbox_l)
tmp_cls_probs.append(cls_prob_l)
tmp_all_scores = np.concatenate((tmp_all_scores, cls_prob_l))
sorted_all_scores = np.sort(tmp_all_scores)[::-1]
keep_num = min(len(sorted_all_scores), max_num)
thresh = max(sorted_all_scores[keep_num - 1], 1e-3)
for label in range(0, n_class):
if label == 0:
continue
bbox_l = tmp_bbox[label - 1]
cls_prob_l = tmp_cls_probs[label - 1]
keep_indices = np.where(cls_prob_l >= thresh)
bbox_l = bbox_l[keep_indices]
cls_prob_l = cls_prob_l[keep_indices]
v_mask_l = np.empty((0, mask_size, mask_size), dtype=np.float32)
v_bbox_l = np.empty((0, 4), dtype=np.float32)
v_cls_prob_l = np.empty((0, ), dtype=np.float32)
for i, bbox in enumerate(bbox_l):
iou = bbox_iou(rois, bbox[np.newaxis, :])
idx = np.where(iou >= mask_merge_thresh)[0]
mask_weights = cls_probs[idx, label]
mask_weights = mask_weights / mask_weights.sum()
mask_prob_l = mask_probs[idx]
rois_l = rois[idx]
clipped_bbox, clipped_mask = mask_aggregation(
rois_l, mask_prob_l, mask_weights, H, W, binary_thresh)
if clipped_bbox is not None and clipped_mask is not None:
clipped_mask = cv2.resize(
clipped_mask.astype(np.float32),
(mask_size, mask_size))
v_mask_l = np.concatenate((v_mask_l, clipped_mask[None]))
v_bbox_l = np.concatenate((v_bbox_l, clipped_bbox[None]))
v_cls_prob_l = np.concatenate(
(v_cls_prob_l, cls_prob_l[i][None]))
keep_indices = v_cls_prob_l > score_thresh
v_mask_l = v_mask_l[keep_indices]
v_bbox_l = v_bbox_l[keep_indices]
v_cls_prob_l = v_cls_prob_l[keep_indices]
v_label_l = np.repeat(label, v_bbox_l.shape[0])
v_masks = np.concatenate((v_masks, v_mask_l))
v_bboxes = np.concatenate((v_bboxes, v_bbox_l))
v_labels = np.concatenate((v_labels, v_label_l))
v_cls_probs = np.concatenate((v_cls_probs, v_cls_prob_l))
return v_bboxes, v_masks, v_labels, v_cls_probs
def decode(self, locs, confs, anchors, in_shape):
"""Decodes back to coordinates of RoIs.
This method decodes :obj:`locs` and :obj:`confs` returned
by a FPN network back to :obj:`rois` and :obj:`roi_indices`.
Args:
locs (list of arrays): A list of arrays whose shape is
:math:`(N, K_l, 4)`, where :math:`N` is the size of batch and
:math:`K_l` is the number of the anchor boxes
of the :math:`l`-th level.
confs (list of arrays): A list of array whose shape is
:math:`(N, K_l)`.
anchors (list of arrays): Anchor boxes returned by :meth:`anchors`.
in_shape (tuple of ints): The shape of input of array
the feature extractor.
Returns:
tuple of two arrays:
:obj:`rois` and :obj:`roi_indices`.
* **rois**: An array of shape :math:`(R, 4)`, \
where :math:`R` is the total number of RoIs in the given batch.
* **roi_indices** : An array of shape :math:`(R,)`.
"""
if chainer.config.train:
nms_limit_pre = self._train_nms_limit_pre
nms_limit_post = self._train_nms_limit_post
else:
nms_limit_pre = self._test_nms_limit_pre
nms_limit_post = self._test_nms_limit_post
rois = []
roi_indices = []
for i in range(in_shape[0]):
roi = []
conf = []
for l in range(len(self._scales)):
loc_l = locs[l].array[i]
conf_l = confs[l].array[i]
roi_l = anchors[l].copy()
# tlbr -> yxhw
roi_l[:, 2:] -= roi_l[:, :2]
roi_l[:, :2] += roi_l[:, 2:] / 2
# offset
roi_l[:, :2] += loc_l[:, :2] * roi_l[:, 2:]
roi_l[:, 2:] *= self.xp.exp(
self.xp.minimum(loc_l[:, 2:], exp_clip))
# yxhw -> tlbr
roi_l[:, :2] -= roi_l[:, 2:] / 2
roi_l[:, 2:] += roi_l[:, :2]
# clip
roi_l[:, :2] = self.xp.maximum(roi_l[:, :2], 0)
roi_l[:, 2:] = self.xp.minimum(roi_l[:, 2:],
self.xp.array(in_shape[2:]))
order = argsort(-conf_l)[:nms_limit_pre]
roi_l = roi_l[order]
conf_l = conf_l[order]
mask = (roi_l[:, 2:] - roi_l[:, :2] > 0).all(axis=1)
roi_l = roi_l[mask]
conf_l = conf_l[mask]
indices = utils.non_maximum_suppression(roi_l,
self._nms_thresh,
limit=nms_limit_post)
roi_l = roi_l[indices]
conf_l = conf_l[indices]
roi.append(roi_l)
conf.append(conf_l)
roi = self.xp.vstack(roi).astype(np.float32)
conf = self.xp.hstack(conf).astype(np.float32)
order = argsort(-conf)[:nms_limit_post]
roi = roi[order]
rois.append(roi)
roi_indices.append(self.xp.array((i, ) * len(roi)))
rois = self.xp.vstack(rois).astype(np.float32)
roi_indices = self.xp.hstack(roi_indices).astype(np.int32)
return rois, roi_indices
