def _retina_detach(self, net_out, index, stride):
key = f'stride{stride}'
A, F = self._num_anchors[key], self._anchors_fpn[key]
[scores, deltas] = map(lambda x: x.asnumpy(), net_out[index:index+2])
height, width = deltas.shape[2], deltas.shape[3]
scores = scores[:, A:, :, :].transpose((0, 2, 3, 1)).reshape((-1, 1))
deltas = deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
anchors = anchors_plane(height, width, stride, F).reshape((-1, 4))
return scores, deltas, anchors
def forward(self, is_train, req, in_data, out_data, aux):
nms = gpu_nms_wrapper(self._threshold, in_data[0][0].context.device_id)
#nms = cpu_nms_wrapper(self._threshold)
cls_prob_dict = dict(zip(self.fpn_keys, in_data[0:len(self.fpn_keys)]))
bbox_pred_dict = dict(zip(self.fpn_keys, in_data[len(self.fpn_keys):2*len(self.fpn_keys)]))
#for i in xrange(6):
# print(i, in_data[i].asnumpy().shape)
batch_size = in_data[0].shape[0]
if batch_size > 1:
raise ValueError("Sorry, multiple images each device is not implemented")
pre_nms_topN = self._rpn_pre_nms_top_n
post_nms_topN = self._rpn_post_nms_top_n
min_size_dict = self._rpn_min_size_fpn
proposals_list = []
scores_list = []
for s in self._feat_stride_fpn:
stride = int(s)
scores = cls_prob_dict['stride%s'%s].asnumpy()[:, self._num_anchors['stride%s'%s]:, :, :]
bbox_deltas = bbox_pred_dict['stride%s'%s].asnumpy()
im_info = in_data[-1].asnumpy()[0, :]
if DEBUG:
print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
print 'scale: {}'.format(im_info[2])
#height, width = int(im_info[0] / stride), int(im_info[1] / stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s'%s]
K = height * width
anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32))
anchors = anchors.reshape((K * A, 4))
#print('pre', bbox_deltas.shape, height, width)
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
#print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
#print(anchors.shape, bbox_deltas.shape, A, K)
proposals = self._bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
proposals = proposals[keep, :]
scores = scores[keep]
proposals_list.append(proposals)
scores_list.append(scores)
proposals = np.vstack(proposals_list)
scores = np.vstack(scores_list)
order = scores.ravel().argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
det = np.hstack((proposals, scores)).astype(np.float32)
if np.shape(det)[0] == 0:
print "Something wrong with the input image(resolution is too low?), generate fake proposals for it."
proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32)
scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32)
det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32)
keep = nms(det)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
if len(keep) < post_nms_topN:
pad = npr.choice(keep, size=post_nms_topN - len(keep))
keep = np.hstack((keep, pad))
proposals = proposals[keep, :]
scores = scores[keep]
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False)))
self.assign(out_data[0], req[0], blob)
if self._output_score:
self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def detect(self, img, threshold=0.5, im_scale=1.0):
proposals_list = []
scores_list = []
landmarks_list = []
data = nd.array(img)
db = mx.io.DataBatch(data=(data, ),
provide_data=[('data', data.shape)])
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
for _idx, s in enumerate(self._feat_stride_fpn):
_key = 'stride%s' % s
stride = int(s)
if self.use_landmarks:
idx = _idx * 3
else:
idx = _idx * 2
scores = net_out[idx].asnumpy()
scores = scores[:, self._num_anchors['stride%s' % s]:, :, :]
idx += 1
bbox_deltas = net_out[idx].asnumpy()
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride, anchors_fpn)
anchors = anchors.reshape((K * A, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
bbox_pred_len = bbox_deltas.shape[3] // A
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, (img.shape[2], img.shape[3]))
scores_ravel = scores.ravel()
order = np.where(scores_ravel >= threshold)[0]
proposals = proposals[order, :]
scores = scores[order]
if stride == 4 and self.decay4 < 1.0:
scores *= self.decay4
proposals[:, 0:4] /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
if not self.vote and self.use_landmarks:
idx += 1
landmark_deltas = net_out[idx].asnumpy()
landmark_deltas = self._clip_pad(landmark_deltas,
(height, width))
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose(
(0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len // 5))
landmarks = self.landmark_pred(anchors, landmark_deltas)
landmarks = landmarks[order, :]
landmarks[:, :, 0:2] /= im_scale
landmarks_list.append(landmarks)
proposals = np.vstack(proposals_list)
landmarks = None
if proposals.shape[0] == 0:
if self.use_landmarks:
landmarks = np.zeros((0, 5, 2))
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
proposals = proposals[order, :]
scores = scores[order]
if not self.vote and self.use_landmarks:
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if not self.vote:
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
if self.use_landmarks:
landmarks = landmarks[keep]
else:
det = np.hstack((pre_det, proposals[:, 4:]))
det = self.bbox_vote(det)
return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False):
#print('in_detect', threshold, scales, do_flip, do_nms)
#print('img_shape: ',img.shape)
proposals_list = []
scores_list = []
landmarks_list = []
timea = datetime.datetime.now()
flips = [0]
if do_flip:
flips = [0, 1]
for im_scale in scales:
for flip in flips:
if im_scale != 1.0:
im = cv2.resize(img,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
else:
im = img.copy()
im = im[:, :, np.newaxis]
if flip:
im = im[:, ::-1, :]
if self.nocrop:
if im.shape[0] % 32 == 0:
h = im.shape[0]
else:
h = (im.shape[0] // 32 + 1) * 32
if im.shape[1] % 32 == 0:
w = im.shape[1]
else:
w = (im.shape[1] // 32 + 1) * 32
_im = np.zeros((h, w, 1), dtype=np.float32)
_im[0:im.shape[0], 0:im.shape[1], :] = im
im = _im
else:
im = im.astype(np.float32)
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X1 uses', diff.total_seconds(), 'seconds')
#self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False)
#im_info = [im.shape[0], im.shape[1], im_scale]
im_info = [im.shape[0], im.shape[1]]
im_tensor = np.zeros((1, 1, im.shape[0], im.shape[1]))
#print('im_tensor_shape: ', im_tensor.shape)
#print('im_shape: ', im.shape)
for i in range(1):
im_tensor[
0,
i, :, :] = (im[:, :, i] / self.pixel_scale -
self.pixel_means[i]) / self.pixel_stds[i]
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X2 uses', diff.total_seconds(), 'seconds')
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data, ),
provide_data=[('data', data.shape)])
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X3 uses', diff.total_seconds(), 'seconds')
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
#print('Len:out: ', len(net_out))
#print('out1: ', net_out[0])
#print('outbbox: ', net_out[1])
#post_nms_topN = self._rpn_post_nms_top_n
#min_size_dict = self._rpn_min_size_fpn
for _idx, s in enumerate(self._feat_stride_fpn):
#if len(scales)>1 and s==32 and im_scale==scales[-1]:
# continue
_key = 'stride%s' % s
stride = int(s)
#if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]):
# continue
if self.use_landmarks:
idx = _idx * 3
else:
idx = _idx * 2
#print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr)
scores = net_out[idx].asnumpy()
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('A uses', diff.total_seconds(), 'seconds')
#print(scores.shape)
#print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:,
self._num_anchors['stride%s' % s]:, :, :]
idx += 1
bbox_deltas = net_out[idx].asnumpy()
#if DEBUG:
# print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
# print 'scale: {}'.format(im_info[2])
#_height, _width = int(im_info[0] / stride), int(im_info[1] / stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
#print('A: ',A)
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride, anchors_fpn)
#print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
#print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr)
#print('HW', (height, width), file=sys.stderr)
#print('anchors_fpn', anchors_fpn.shape, file=sys.stderr)
#print('anchors', anchors.shape, file=sys.stderr)
#print('bbox_deltas', bbox_deltas.shape, file=sys.stderr)
#print('scores', scores.shape, file=sys.stderr)
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
#print('pre', bbox_deltas.shape, height, width)
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
#print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
#print('bbox_deltas.shape[3]:',bbox_deltas.shape[3])
bbox_pred_len = bbox_deltas.shape[3] // A
#print('bbox_deltas.shape:',bbox_deltas.shape)
#print('boxlen:',bbox_pred_len)
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
#print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
#if self.vote:
# if im_scale>1.0:
# keep = self._filter_boxes2(proposals, 160*im_scale, -1)
# else:
# keep = self._filter_boxes2(proposals, -1, 100*im_scale)
# if stride==4:
# keep = self._filter_boxes2(proposals, 12*im_scale, -1)
# proposals = proposals[keep, :]
# scores = scores[keep]
#keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
#proposals = proposals[keep, :]
#scores = scores[keep]
#print('333', proposals.shape)
scores_ravel = scores.ravel()
#print('__shapes', proposals.shape, scores_ravel.shape)
#print('max score', np.max(scores_ravel))
order = np.where(scores_ravel >= threshold)[0]
#print('order:',order)
#print('score_order:',scores_ravel[order])
#_scores = scores_ravel[order]
#_order = _scores.argsort()[::-1]
#order = order[_order]
#order = [50]
proposals = proposals[order, :]
scores = scores[order]
if stride == 4 and self.decay4 < 1.0:
scores *= self.decay4
if flip:
oldx1 = proposals[:, 0].copy()
oldx2 = proposals[:, 2].copy()
proposals[:, 0] = im.shape[1] - oldx2 - 1
proposals[:, 2] = im.shape[1] - oldx1 - 1
proposals[:, 0:4] /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
if not self.vote and self.use_landmarks:
idx += 1
landmark_deltas = net_out[idx].asnumpy()
landmark_deltas = self._clip_pad(
landmark_deltas, (height, width))
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose(
(0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
#print(landmark_deltas.shape, landmark_deltas)
landmarks = self.landmark_pred(anchors,
landmark_deltas)
landmarks = landmarks[order, :]
if flip:
landmarks[:, :,
0] = im.shape[1] - landmarks[:, :, 0] - 1
#for a in range(5):
# oldx1 = landmarks[:, a].copy()
# landmarks[:,a] = im.shape[1] - oldx1 - 1
order = [1, 0, 2, 4, 3]
flandmarks = landmarks.copy()
for idx, a in enumerate(order):
flandmarks[:, idx, :] = landmarks[:, a, :]
#flandmarks[:, idx*2] = landmarks[:,a*2]
#flandmarks[:, idx*2+1] = landmarks[:,a*2+1]
landmarks = flandmarks
landmarks[:, :, 0:2] /= im_scale
#landmarks /= im_scale
#landmarks = landmarks.reshape( (-1, landmark_pred_len) )
landmarks_list.append(landmarks)
#proposals = np.hstack((proposals, landmarks))
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('B uses', diff.total_seconds(), 'seconds')
proposals = np.vstack(proposals_list)
landmarks = None
if proposals.shape[0] == 0:
if self.use_landmarks:
landmarks = np.zeros((0, 5, 2))
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
#print('shapes', proposals.shape, scores.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
#if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
proposals = proposals[order, :]
scores = scores[order]
if not self.vote and self.use_landmarks:
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if not self.vote:
#print('pre_det_type: ', type(pre_det))
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
if self.use_landmarks:
landmarks = landmarks[keep]
else:
det = np.hstack((pre_det, proposals[:, 4:]))
det = self.bbox_vote(det)
#if self.use_landmarks:
# det = np.hstack((det, landmarks))
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('C uses', diff.total_seconds(), 'seconds')
return det, scores #landmarks
def detect(self, img, scales_index=0):
proposals_list = []
scores_list = []
im_src = img.copy()
CONSTANT = config.TEST.CONSTANT
BLACK = [0, 0, 0]
img = cv2.copyMakeBorder(img, CONSTANT, CONSTANT, CONSTANT, CONSTANT, cv2.BORDER_CONSTANT, value=BLACK)
scales = self.get_boxes(img, scales_index)
for im_scale in scales:
if im_scale != 1.0:
im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR)
else:
im = img
im = im.astype(np.float32)
# self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False)
im_info = [im.shape[0], im.shape[1], im_scale]
im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1]))
for i in range(3):
im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i]
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)])
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
pre_nms_topN = self._rpn_pre_nms_top_n
for s in self._feat_stride_fpn:
if len(scales) > 1 and s == 32 and im_scale == scales[-1]:
continue
_key = 'stride%s' % s
stride = int(s)
idx = 0
if s == 16:
idx = 2
elif s == 8:
idx = 4
# print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr)
scores = net_out[idx].asnumpy()
# print(scores.shape)
idx += 1
# print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:, self._num_anchors['stride%s' % s]:, :, :]
bbox_deltas = net_out[idx].asnumpy()
_height, _width = int(im_info[0] / stride), int(im_info[1] / stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32))
# print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
# print('pre', bbox_deltas.shape, height, width)
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
# print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
# print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self._bbox_pred(anchors, bbox_deltas)
# proposals = anchors
proposals = clip_boxes(proposals, im_info[:2])
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
proposals /= im_scale
# #add by sai with pyramidbox to filt scale face
# if im_scale > 1:
# index = np.where(
# np.minimum(proposals[:, 2] - proposals[:, 0] + 1,
# proposals[:, 3] - proposals[:, 1] + 1) < 50)[0]
# proposals = proposals[index, :]
# scores = scores[index, :]
# else:
# index = np.where(
# np.maximum(proposals[:, 2] - proposals[:, 0] + 1,
# proposals[:, 3] - proposals[:, 1] + 1) > 20)[0]
# proposals = proposals[index, :]
# scores = scores[index, :]
proposals_list.append(proposals)
scores_list.append(scores)
proposals = np.vstack(proposals_list)
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
proposals = proposals[order, :]
scores = scores[order]
det = np.hstack((proposals, scores)).astype(np.float32)
if self.nms_threshold < 1.0:
keep = self.nms(det)
det = det[keep, :]
threshold = config.TEST.SCORE_THRESH
if threshold > 0.0:
keep = np.where(det[:, 4] >= threshold)[0]
det = det[keep, :]
# add by sai
if det.shape[0] != 0:
for i in range(det.shape[0]):
det[i, :][0] = det[i, :][0] - CONSTANT
det[i, :][1] = det[i, :][1] - CONSTANT
det[i, :][2] = det[i, :][2] - CONSTANT
det[i, :][3] = det[i, :][3] - CONSTANT
if det[i, :][0] < 0:
det[i, :][0] = 0
if det[i, :][2] > im_src.shape[1]:
det[i, :][2] = im_src.shape[1]
if det[i, :][1] < 0:
det[i, :][1] = 0
if det[i, :][3] > im_src.shape[0]:
det[i, :][3] = im_src.shape[0]
return det
def postprocess(net_out, threshold, ctx_id, im_scale, im_info):
# im_info = [640, 640]
flip = False
decay4 = 0.5
vote = False
fpn_keys = []
anchor_cfg = None
bbox_stds = [1.0, 1.0, 1.0, 1.0]
# im_scale = 1.0
landmark_std = 1.0
nms_threshold = 0.4
proposals_list = []
scores_list = []
landmarks_list = []
strides_list = []
use_landmarks = True
if ctx_id >= 0:
nms = gpu_nms_wrapper(nms_threshold, ctx_id)
else:
nms = cpu_nms_wrapper(nms_threshold)
use_landmarks = True
_ratio = (1., )
_feat_stride_fpn = [32, 16, 8]
anchor_cfg = {
'32': {
'SCALES': (32, 16),
'BASE_SIZE': 16,
'RATIOS': _ratio,
'ALLOWED_BORDER': 9999
},
'16': {
'SCALES': (8, 4),
'BASE_SIZE': 16,
'RATIOS': _ratio,
'ALLOWED_BORDER': 9999
},
'8': {
'SCALES': (2, 1),
'BASE_SIZE': 16,
'RATIOS': _ratio,
'ALLOWED_BORDER': 9999
},
}
for s in _feat_stride_fpn:
fpn_keys.append('stride%s' % s)
dense_anchor = False
_anchors_fpn = dict(
zip(fpn_keys,
generate_anchors_fpn(dense_anchor=dense_anchor, cfg=anchor_cfg)))
for k in _anchors_fpn:
v = _anchors_fpn[k].astype(np.float32)
_anchors_fpn[k] = v
_num_anchors = dict(
zip(fpn_keys, [anchors.shape[0] for anchors in _anchors_fpn.values()]))
sym_idx = 0
for _idx, s in enumerate(_feat_stride_fpn):
# print(sym_idx)
_key = 'stride%s' % s
# print(_key)
stride = int(s)
scores = net_out[sym_idx] #.asnumpy()
scores = scores[:, _num_anchors['stride%s' % s]:, :, :]
bbox_deltas = net_out[sym_idx + 1] # .asnumpy()
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = _num_anchors['stride%s' % s]
K = height * width
anchors_fpn = _anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride, anchors_fpn)
anchors = anchors.reshape((K * A, 4))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
bbox_pred_len = bbox_deltas.shape[3] // A
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * bbox_stds[0]
bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * bbox_stds[1]
bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * bbox_stds[2]
bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * bbox_stds[3]
proposals = bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
if stride == 4 and decay4 < 1.0:
scores *= decay4
scores_ravel = scores.ravel()
order = np.where(scores_ravel >= threshold)[0]
proposals = proposals[order, :]
scores = scores[order]
if flip:
oldx1 = proposals[:, 0].copy()
oldx2 = proposals[:, 2].copy()
proposals[:, 0] = im.shape[1] - oldx2 - 1
proposals[:, 2] = im.shape[1] - oldx1 - 1
#proposals[:,0:4] /= im_scale
#print(proposals[:,0])
proposals[:, 0] /= im_scale[0]
#print(pp)
proposals[:, 1] /= im_scale[1]
proposals[:, 2] /= im_scale[0]
proposals[:, 3] /= im_scale[1]
#print(proposals[:,0])
proposals_list.append(proposals)
scores_list.append(scores)
if nms_threshold < 0.0:
_strides = np.empty(shape=(scores.shape), dtype=np.float32)
_strides.fill(stride)
strides_list.append(_strides)
if not vote and use_landmarks:
landmark_deltas = net_out[sym_idx + 2] #.asnumpy()
# print(landmark_deltas)
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
landmark_deltas *= landmark_std
landmarks = landmark_pred(anchors, landmark_deltas)
landmarks = landmarks[order, :]
if flip:
landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1
order = [1, 0, 2, 4, 3]
flandmarks = landmarks.copy()
for idx, a in enumerate(order):
flandmarks[:, idx, :] = landmarks[:, a, :]
landmarks = flandmarks
landmarks[:, :, 0:2] /= im_scale
landmarks_list.append(landmarks)
if use_landmarks:
sym_idx += 3
else:
sym_idx += 2
proposals = np.vstack(proposals_list)
landmarks = None
if proposals.shape[0] == 0:
if use_landmarks:
landmarks = np.zeros((0, 5, 2))
if nms_threshold < 0.0:
return np.zeros((0, 6)), landmarks
else:
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
proposals = proposals[order, :]
scores = scores[order]
if nms_threshold < 0.0:
strides = np.vstack(strides_list)
strides = strides[order]
if not vote and use_landmarks:
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
if nms_threshold > 0.0:
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if not vote:
keep = nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
if use_landmarks:
landmarks = landmarks[keep]
else:
det = np.hstack((pre_det, proposals[:, 4:]))
det = bbox_vote(det, nms_threshold)
elif nms_threshold < 0.0:
det = np.hstack((proposals[:,
0:4], scores, strides)).astype(np.float32,
copy=False)
else:
det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
return det, landmarks
def detect(self, img, threshold=0.5):
"""
Detect all the faces and landmarks in an image
:param img: input image
:param threshold: detection threshold
:return: tuple faces, landmarks
"""
proposals_list = []
scores_list = []
landmarks_list = []
im_tensor, im_info, im_scale = self._preprocess_image(img)
net_out = self.model(im_tensor)
net_out = [elt.numpy() for elt in net_out]
sym_idx = 0
for _idx, s in enumerate(self._feat_stride_fpn):
_key = 'stride%s' % s
scores = net_out[sym_idx]
scores = scores[:, :, :, self._num_anchors['stride%s' % s]:]
bbox_deltas = net_out[sym_idx + 1]
height, width = bbox_deltas.shape[1], bbox_deltas.shape[2]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, s, anchors_fpn)
anchors = anchors.reshape((K * A, 4))
scores = scores.reshape((-1, 1))
bbox_deltas = bbox_deltas
bbox_pred_len = bbox_deltas.shape[3] // A
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * self.bbox_stds[0]
bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * self.bbox_stds[1]
bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * self.bbox_stds[2]
bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * self.bbox_stds[3]
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
if s == 4 and self.decay4 < 1.0:
scores *= self.decay4
scores_ravel = scores.ravel()
order = np.where(scores_ravel >= threshold)[0]
proposals = proposals[order, :]
scores = scores[order]
proposals[:, 0:4] /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
landmark_deltas = net_out[sym_idx + 2]
landmark_pred_len = landmark_deltas.shape[3] // A
landmark_deltas = landmark_deltas.reshape(
(-1, 5, landmark_pred_len // 5))
landmarks = self.landmark_pred(anchors, landmark_deltas)
landmarks = landmarks[order, :]
landmarks[:, :, 0:2] /= im_scale
landmarks_list.append(landmarks)
sym_idx += 3
proposals = np.vstack(proposals_list)
if proposals.shape[0] == 0:
landmarks = np.zeros((0, 5, 2))
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
proposals = proposals[order, :]
scores = scores[order]
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
landmarks = landmarks[keep]
return det, landmarks
def detect(self,img, threshold=0.05):
image_size = (img.shape[0], img.shape[1])
#self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False)
im_info = [image_size[0], image_size[1], 1.0]
data = nd.zeros( (1 ,3, image_size[0], image_size[1]) )
nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
nimg = np.transpose(nimg, (2,0,1))
nimg = nd.array(nimg)
data[0] = nimg
db = mx.io.DataBatch(data=(data,))
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
pre_nms_topN = self._rpn_pre_nms_top_n
#post_nms_topN = self._rpn_post_nms_top_n
#min_size_dict = self._rpn_min_size_fpn
proposals_list = []
scores_list = []
idx = 0
for s in self._feat_stride_fpn:
_key = 'stride%s'%s
stride = int(s)
scores = net_out[idx].asnumpy()
#print(scores.shape)
idx+=1
#print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:, self._num_anchors['stride%s'%s]:, :, :]
bbox_deltas = net_out[idx].asnumpy()
idx+=1
#if DEBUG:
# print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
# print 'scale: {}'.format(im_info[2])
_height, _width = int(im_info[0] / stride), int(im_info[1] / stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s'%s]
K = height * width
anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32))
print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
#print('pre', bbox_deltas.shape, height, width)
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
#print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
#print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self._bbox_pred(anchors, bbox_deltas)
#proposals = anchors
proposals = clip_boxes(proposals, im_info[:2])
#keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
#proposals = proposals[keep, :]
#scores = scores[keep]
#print('333', proposals.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
proposals_list.append(proposals)
scores_list.append(scores)
proposals = np.vstack(proposals_list)
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
#if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
#if pre_nms_topN > 0:
# order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
det = np.hstack((proposals, scores)).astype(np.float32)
#if np.shape(det)[0] == 0:
# print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.")
# proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32)
# scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32)
# det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32)
if self.nms_threshold<1.0:
keep = self.nms(det)
det = det[keep, :]
if threshold>0.0:
keep = np.where(det[:, 4] >= threshold)[0]
det = det[keep, :]
return det
def detect(self, img, threshold=0.5):
proposals_list = []
scores_list = []
landmarks_list = []
im_info = [640, 640]
if img.shape[0] != img.shape[1]:
BLUE = (255, 0, 0)
if img.shape[0] > img.shape[1]:
img = cv2.copyMakeBorder(img,
0,
0,
0,
img.shape[0] - img.shape[1],
cv2.BORDER_CONSTANT,
value=BLUE)
else:
img = cv2.copyMakeBorder(img,
0,
img.shape[1] - img.shape[0],
0,
0,
cv2.BORDER_CONSTANT,
value=BLUE)
re_scale = float(im_info[0]) / float(img.shape[0])
img = cv2.resize(img, (im_info[0], im_info[1]))
img = img.astype(np.float32)
im_tensor = np.zeros((1, 3, img.shape[0], img.shape[1]))
for i in range(3):
im_tensor[
0,
i, :, :] = (img[:, :, 2 - i] / self.pixel_scale -
self.pixel_means[2 - i]) / self.pixel_stds[2 - i]
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data, ),
provide_data=[('data', data.shape)])
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
for _idx, s in enumerate(self._feat_stride_fpn):
stride = int(s)
idx = _idx * 3
scores = net_out[idx].asnumpy()
scores = scores[:, self._num_anchors['stride%s' % s]:, :, :]
idx += 1
bbox_deltas = net_out[idx].asnumpy()
idx += 1
landmark_deltas = net_out[idx].asnumpy()
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride, anchors_fpn)
anchors = anchors.reshape((K * A, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
bbox_pred_len = bbox_deltas.shape[3] // A
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
landmark_deltas = self._clip_pad(landmark_deltas, (height, width))
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
landmarks = self.landmark_pred(anchors, landmark_deltas)
scores_ravel = scores.ravel()
order = np.where(scores_ravel >= threshold)[0]
scores = scores[order]
proposals = proposals[order, :]
landmarks = landmarks[order, :]
if stride == 4 and self.decay4 < 1.0:
scores *= self.decay4
proposals[:, 0:4] /= re_scale
landmarks[:, :, 0:2] /= re_scale
scores_list.append(scores)
proposals_list.append(proposals)
landmarks_list.append(landmarks)
scores = np.vstack(scores_list)
proposals = np.vstack(proposals_list)
landmarks = np.vstack(landmarks_list)
if proposals.shape[0] == 0:
return np.zeros((0, 5)), np.zeros((0, 5, 2))
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
scores = scores[order]
proposals = proposals[order, :]
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
landmarks = landmarks[keep]
return det, landmarks
def detect(self, img, threshold=0.05, scales=[1.0]):
proposals_list = []
scores_list = []
for im_scale in scales:
if im_scale!=1.0:
im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR)
else:
im = img
im = im.astype(np.float32)
#self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False)
im_info = [im.shape[0], im.shape[1], im_scale]
im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1]))
for i in range(3):
im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i]
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)])
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
pre_nms_topN = self._rpn_pre_nms_top_n
#post_nms_topN = self._rpn_post_nms_top_n
#min_size_dict = self._rpn_min_size_fpn
for s in self._feat_stride_fpn:
if len(scales)>1 and s==32 and im_scale==scales[-1]:
continue
_key = 'stride%s'%s
stride = int(s)
idx = 0
if s==16:
idx=2
elif s==8:
idx=4
print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr)
scores = net_out[idx].asnumpy()
#print(scores.shape)
idx+=1
#print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:, self._num_anchors['stride%s'%s]:, :, :]
bbox_deltas = net_out[idx].asnumpy()
#if DEBUG:
# print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
# print 'scale: {}'.format(im_info[2])
_height, _width = int(im_info[0] / stride), int(im_info[1] / stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s'%s]
K = height * width
anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32))
#print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
#print('pre', bbox_deltas.shape, height, width)
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
#print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
#print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self._bbox_pred(anchors, bbox_deltas)
#proposals = anchors
proposals = clip_boxes(proposals, im_info[:2])
#keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
#proposals = proposals[keep, :]
#scores = scores[keep]
#print('333', proposals.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
proposals /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
proposals = np.vstack(proposals_list)
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
#if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
#if pre_nms_topN > 0:
# order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
det = np.hstack((proposals, scores)).astype(np.float32)
#if np.shape(det)[0] == 0:
# print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.")
# proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32)
# scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32)
# det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32)
if self.nms_threshold<1.0:
keep = self.nms(det)
det = det[keep, :]
if threshold>0.0:
keep = np.where(det[:, 4] >= threshold)[0]
det = det[keep, :]
return det
def detect(self, img, thresh, scales=[1.0], do_flip=False):
proposal_list = []
scores_list = []
landmarks_list = []
flips = [0,1] if do_flip else [0]
for im_scale in scales:
for flip in flips:
if im_scale!=1.0:
img = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR)
else:
img = img.copy()
if flip:
img = img[:,::-1,:]
# img = self.pad_img_to_32(img)
imgshape = [img.shape[0], img.shape[1]]
img = self.preprocess(img)
img = torch.from_numpy(img)
if self.use_gpu:
img = img.cuda()
net_out = self.model(img)
for _idx,s in enumerate(self.fpn_keys):
idx = _idx * 3
scores = net_out[idx].detach().cpu().numpy()
scores = scores[:, self._num_anchors[s]:]
idx += 1
bbox_deltas = net_out[idx].detach().cpu().numpy()
h, w = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors[s]
K = h*w
anchors_fpn = self._anchors_fpn[s]
anchors_fpn = np.float32(anchors_fpn)
anchors = anchors_plane(h, w, s, anchors_fpn)
anchors = anchors.reshape((K*A, 4))
scores = self._clip_pad(scores, (h, w))
scores = scores.transpose([0,2,3,1]).reshape([-1,1])
# print('SCR')
# print(scores)
# print(scores.shape)
# input()
bbox_deltas = self._clip_pad(bbox_deltas, (h,w))
bbox_deltas = bbox_deltas.transpose([0,2,3,1])
bbox_pred_len = bbox_deltas.shape[3]//A
bbox_deltas = bbox_deltas.reshape([-1, bbox_pred_len])
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, imgshape)
scores_ravel = scores.ravel()
order = np.where(scores_ravel>=thresh)[0]
proposals = proposals[order]
scores = scores[order]
if flip:
oldx1 = proposals[:, 0].copy()
oldx2 = proposals[:, 2].copy()
proposals[:, 0] = im.shape[1] - oldx2 - 1
proposals[:, 2] = im.shape[1] - oldx1 - 1
proposals[:,:4] /= im_scale
# print('proposals')
# print(proposals)
# print(proposals.shape)
# input()
proposal_list.append(proposals)
scores_list.append(scores)
# landmarks
idx += 1
landmark_deltas = net_out[idx].detach().cpu().numpy()
landmark_deltas = self._clip_pad(landmark_deltas, (h,w))
landmark_pred_len = landmark_deltas.shape[1]//A
landmark_deltas = landmark_deltas.transpose((0,2,3,1)).reshape([-1,5,landmark_pred_len//5])
landmarks = self.landmark_pred(anchors, landmark_deltas)
landmarks = landmarks[order, :]
if flip:
landmarks[:,:,0] = imgshape[1] - landmarks[:,:,0] - 1
order = [1,0,2,4,3]
flandmarks = landmarks[:,np.int32(order)]
landmarks[:,:,:2] /= im_scale
landmarks_list.append(landmarks)
# print('PROPOSAL', proposal_list)
proposals = np.vstack(proposal_list)
landmarks = None
if proposals.shape[0]==0:
return np.zeros([0,5]), np.zeros([0,5,2])
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
proposals = proposals[order]
scores = scores[order]
landmarks = np.vstack(landmarks_list)
landmarks = np.float32(landmarks[order])
pre_det = np.hstack([proposals[:, 0:4], scores])
pre_det = np.float32(pre_det)
keep = self.nms(pre_det)
det = np.hstack([pre_det, proposals[:,4:]])
det = det[keep]
landmarks = landmarks[keep]
return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False):
print(
'get into detect, confi thresold={}, scales={}, do_flip={}'.format(
threshold, scales, do_flip))
proposals_list = []
scores_list = []
landmarks_list = []
timea = datetime.datetime.now()
flips = [0]
if do_flip:
flips = [0, 1]
#TODO 根据scale给输入的图片做resize
for im_scale in scales:
for flip in flips:
if im_scale != 1.0:
im = cv2.resize(img,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
else:
im = img.copy()
# 对图像做翻转
if flip:
im = im[:, ::-1, :]
# 对图像做裁剪
if self.nocrop:
if im.shape[0] % 32 == 0:
h = im.shape[0]
else:
h = (im.shape[0] // 32 + 1) * 32
if im.shape[1] % 32 == 0:
w = im.shape[1]
else:
w = (im.shape[1] // 32 + 1) * 32
_im = np.zeros((h, w, 3), dtype=np.float32)
_im[0:im.shape[0], 0:im.shape[1], :] = im
im = _im
else:
im = im.astype(np.float32)
im_info = [im.shape[0], im.shape[1]] #h,w
im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1]))
for i in range(3):
im_tensor[0, i, :, :] = (
im[:, :, 2 - i] / self.pixel_scale -
self.pixel_means[2 - i]) / self.pixel_stds[
2 - i] #TODO 这里好像将Channel顺序倒过来了,与image.py保持一致
data = np.array(im_tensor)
# 读入模型进行推理，得到预测值
net_out = self.get_pred(data)
if self.debug:
for key in net_out.keys():
print('{} = {}\n'.format(key, net_out[key].shape))
for _idx, s in enumerate(self._feat_stride_fpn):
# print('begin stride{}-------------------------------------------------\n'.format(s))
_key = 'stride%s' % s
stride = int(s)
# print('getting im_scale={}, stride={}, len(net_out)={}, data.shape={}'.format(im_scale, stride, len(net_out), data.shape))
scores = net_out['rpn_cls_prob_stride%s' %
s] #TODO 要注意这里是nhwc不是nchw
if self.debug:
print('get score:', scores.shape)
# print('stride{}: scores before shape={}, idx={}'.format(stride, scores.shape, self._num_anchors['stride%s' % s]))
scores = scores[:, 1].reshape(
(-1, 1)) #TODO: (H*W*A, 1) #这里的1表示正类的概率
if self.debug:
print('AAAAstride{}: scores after shape={}'.format(
stride, scores.shape))
bbox_deltas = net_out['rpn_bbox_pred_stride%s' %
s] #TODO NHW8
height, width = bbox_deltas.shape[1], bbox_deltas.shape[2]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride,
anchors_fpn) #获取该特征图上的所有anchor
#print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
if self.debug:
print('HW', (height, width))
print('anchors_fpn', anchors_fpn)
print('anchors', anchors.shape, '\n')
# scores = self._clip_pad_NCHW(scores, (height, width)) #(1, 4, H, W)
# scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #(1, H, W, 4)
# print('scores reshape', scores.shape)
if self.debug:
print('before bbox_deltas', bbox_deltas.shape)
bbox_deltas = self._clip_pad_NHWC(
bbox_deltas, (height, width)) #(1, H, W, 8)
# bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))#(1, H, W, 8)
bbox_pred_len = bbox_deltas.shape[3] // A #4
bbox_deltas = bbox_deltas.reshape(
(-1, bbox_pred_len)) #(H*W*2, 4)
if self.debug:
print('after bbox_deltas', bbox_deltas.shape, height,
width, '\n')
#print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self.bbox_pred(
anchors,
bbox_deltas) #TODO important! 将anchor加上delta进行处理
proposals = clip_boxes(proposals, im_info[:2])
scores_ravel = scores.ravel()
max_score = np.max(scores_ravel)
print('proposals.shape={}, score_ravel.shape={}'.format(
proposals.shape, scores_ravel.shape))
print('max score', max_score)
order = np.where(scores_ravel >= threshold)[0]
#_scores = scores_ravel[order]
#_order = _scores.argsort()[::-1]
#order = order[_order]
proposals = proposals[order, :]
scores = scores[order]
if flip:
oldx1 = proposals[:, 0].copy()
oldx2 = proposals[:, 2].copy()
proposals[:, 0] = im.shape[1] - oldx2 - 1
proposals[:, 2] = im.shape[1] - oldx1 - 1
proposals[:, 0:
4] /= im_scale #TODO important 在这里将找到的proposal给映射回原来图像的位置
proposals_list.append(proposals)
scores_list.append(scores)
if not self.vote and self.use_landmarks:
landmark_deltas = net_out['rpn_landmark_pred_stride%s'
% s] #(1,20,H,W)
if self.debug:
print('before landmark_deltas',
landmark_deltas.shape)
landmark_deltas = self._clip_pad_NCHW(
landmark_deltas, (height, width))
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose(
(0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
if self.debug:
print('after landmark_deltas',
landmark_deltas.shape, landmark_deltas)
landmarks = self.landmark_pred(anchors,
landmark_deltas)
landmarks = landmarks[order, :]
if flip:
landmarks[:, :,
0] = im.shape[1] - landmarks[:, :, 0] - 1
#for a in range(5):
# oldx1 = landmarks[:, a].copy()
# landmarks[:,a] = im.shape[1] - oldx1 - 1
order = [1, 0, 2, 4, 3]
flandmarks = landmarks.copy()
for idx, a in enumerate(order):
flandmarks[:, idx, :] = landmarks[:, a, :]
#flandmarks[:, idx*2] = landmarks[:,a*2]
#flandmarks[:, idx*2+1] = landmarks[:,a*2+1]
landmarks = flandmarks
landmarks[:, :, 0:2] /= im_scale
landmarks_list.append(landmarks)
if self.debug:
print(
'end stride{}-------------------------------------------------\n'
.format(s))
proposals = np.vstack(proposals_list)
landmarks = None
if proposals.shape[0] == 0:
if self.use_landmarks:
landmarks = np.zeros((0, 5, 2))
return np.zeros((0, 5)), landmarks
# for i in range(len(scores_list)):
# print('hhhhh score,shape=',scores_list[i].shape)
scores = np.vstack(scores_list)
print('finally!!! proposals.shape={}, score.shape={}'.format(
proposals.shape, scores.shape))
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1] # 按照score从大到小排序
proposals = proposals[order, :]
scores = scores[order]
if self.debug:
print('sort score=', scores)
if not self.vote and self.use_landmarks:
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if not self.vote:
print('begin to NMS!!\n')
keep = self.nms(pre_det)
# print('before hstack: pre_det={}, proposals.shape={}, proposals[:,4:]={}'.format(pre_det.shape, proposals.shape, proposals[:,4:]))
det = np.hstack((pre_det, proposals[:, 4:]))
# print('after hstack: pre_det={}, proposals.shape={}'.format(pre_det.shape, proposals.shape))
det = det[keep, :]
if self.use_landmarks:
landmarks = landmarks[keep]
else:
det = np.hstack((pre_det, proposals[:, 4:]))
det = self.bbox_vote(det)
return det, landmarks
def forward(self, is_train, req, in_data, out_data, aux):
nms = gpu_nms_wrapper(self._threshold, in_data[0][0].context.device_id)
cls_prob_dict = dict(zip(self.fpn_keys, in_data[0:len(self.fpn_keys)]))
bbox_pred_dict = dict(
zip(self.fpn_keys,
in_data[len(self.fpn_keys):2 * len(self.fpn_keys)]))
#for i in xrange(6):
# print(i, in_data[i].asnumpy().shape)
batch_size = in_data[0].shape[0]
if batch_size > 1:
raise ValueError(
"Sorry, multiple images each device is not implemented")
pre_nms_topN = self._rpn_pre_nms_top_n
post_nms_topN = self._rpn_post_nms_top_n
min_size_dict = self._rpn_min_size_fpn
proposals_list = []
scores_list = []
for s in self._feat_stride_fpn:
_key = 'stride%s' % s
stride = int(s)
scores = cls_prob_dict[_key].asnumpy()
#print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:, self._num_anchors['stride%s' % s]:, :, :]
bbox_deltas = bbox_pred_dict['stride%s' % s].asnumpy()
im_info = in_data[-1].asnumpy()[0, :]
#if DEBUG:
# print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
# print 'scale: {}'.format(im_info[2])
_height, _width = int(im_info[0] / stride), int(im_info[1] /
stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors = anchors_plane(
height, width, stride,
self._anchors_fpn['stride%s' % s].astype(np.float32))
print((height, width), (_height, _width),
anchors.shape,
bbox_deltas.shape,
scores.shape,
file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
#print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self._bbox_pred(anchors, bbox_deltas)
#proposals = anchors
proposals = clip_boxes(proposals, im_info[:2])
#keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
#proposals = proposals[keep, :]
#scores = scores[keep]
#print('333', proposals.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
proposals_list.append(proposals)
scores_list.append(scores)
proposals = np.vstack(proposals_list)
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
#if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
#if pre_nms_topN > 0:
# order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
det = np.hstack((proposals, scores)).astype(np.float32)
#if np.shape(det)[0] == 0:
# print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.")
# proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32)
# scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32)
# det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32)
if self._threshold < 1.0:
keep = nms(det)
else:
keep = range(det.shape[0])
#print(det.shape, len(keep), post_nms_topN)
if post_nms_topN > 0:
keep = keep[:post_nms_topN]
#print(det.shape, len(keep), post_nms_topN)
num_keep = len(keep)
#print('keep', keep, file=sys.stderr)
if len(keep) < post_nms_topN:
pad = npr.choice(keep, size=post_nms_topN - len(keep))
keep = np.hstack((keep, pad))
proposals = proposals[keep, :]
scores = scores[keep]
scores[num_keep:, :] = -1.0
#print('333 proposals', proposals[0:5,:], file=sys.stderr)
#print('det', det.shape, num_keep)
#print('first proposal', proposals[0], file=sys.stderr)
batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32)
blob = np.hstack((batch_inds, proposals.astype(np.float32,
copy=False)))
self.assign(out_data[0], req[0], blob)
if self._output_score:
self.assign(out_data[1], req[1],
scores.astype(np.float32, copy=False))
def detect(self, img, threshold=0.05, scales=[1.0]):
proposals_list = []
scores_list = []
for im_scale in scales:
if im_scale != 1.0:
im = cv2.resize(img,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
else:
im = img
im = im.astype(np.float32)
# self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False)
im_info = [im.shape[0], im.shape[1], im_scale]
im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1]))
for i in range(3):
im_tensor[0,
i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i]
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data, ),
provide_data=[('data', data.shape)])
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
pre_nms_topN = self._rpn_pre_nms_top_n
# post_nms_topN = self._rpn_post_nms_top_n
# min_size_dict = self._rpn_min_size_fpn
for s in self._feat_stride_fpn:
if len(scales) > 1 and s == 32 and im_scale == scales[-1]:
continue
_key = 'stride%s' % s
stride = int(s)
idx = 0
if s == 16:
idx = 2
elif s == 8:
idx = 4
print('getting',
im_scale,
stride,
idx,
len(net_out),
data.shape,
file=sys.stderr)
scores = net_out[idx].asnumpy()
# print(scores.shape)
idx += 1
# print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:, self._num_anchors['stride%s' % s]:, :, :]
bbox_deltas = net_out[idx].asnumpy()
# if DEBUG:
# print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
# print 'scale: {}'.format(im_info[2])
_height, _width = int(im_info[0] / stride), int(im_info[1] /
stride)
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors = anchors_plane(
height, width, stride,
self._anchors_fpn['stride%s' % s].astype(np.float32))
# print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
# print('pre', bbox_deltas.shape, height, width)
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
# print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape(
(-1, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
# print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
proposals = self._bbox_pred(anchors, bbox_deltas)
# proposals = anchors
proposals = clip_boxes(proposals, im_info[:2])
# keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
# proposals = proposals[keep, :]
# scores = scores[keep]
# print('333', proposals.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
if pre_nms_topN > 0:
order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
proposals /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
proposals = np.vstack(proposals_list)
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
# if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
# if pre_nms_topN > 0:
# order = order[:pre_nms_topN]
proposals = proposals[order, :]
scores = scores[order]
det = np.hstack((proposals, scores)).astype(np.float32)
# if np.shape(det)[0] == 0:
# print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.")
# proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32)
# scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32)
# det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32)
if self.nms_threshold < 1.0:
keep = self.nms(det)
det = det[keep, :]
if threshold > 0.0:
keep = np.where(det[:, 4] >= threshold)[0]
det = det[keep, :]
return det
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False):
proposals_list = []
scores_list = []
landmarks_list = []
timea = datetime.datetime.now()
flips = [0]
if do_flip:
flips = [0, 1]
for im_scale in scales:
for flip in flips:
if im_scale != 1.0:
im = cv2.resize(img,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
else:
im = img.copy()
if flip:
im = im[:, ::-1, :]
if self.nocrop:
if im.shape[0] % 32 == 0:
h = im.shape[0]
else:
h = (im.shape[0] // 32 + 1) * 32
if im.shape[1] % 32 == 0:
w = im.shape[1]
else:
w = (im.shape[1] // 32 + 1) * 32
_im = np.zeros((h, w, 3), dtype=np.float32)
_im[0:im.shape[0], 0:im.shape[1], :] = im
im = _im
else:
im = im.astype(np.float32)
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X1 uses', diff.total_seconds(), 'seconds')
im_info = [im.shape[0], im.shape[1]]
im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1]))
for i in range(3):
im_tensor[0, i, :, :] = (
im[:, :, 2 - i] / self.pixel_scale -
self.pixel_means[2 - i]) / self.pixel_stds[2 - i]
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X2 uses', diff.total_seconds(), 'seconds')
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data, ),
provide_data=[('data', data.shape)])
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X3 uses', diff.total_seconds(), 'seconds')
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
for _idx, s in enumerate(self._feat_stride_fpn):
_key = 'stride%s' % s
stride = int(s)
if self.use_landmarks:
idx = _idx * 3
else:
idx = _idx * 2
scores = net_out[idx].asnumpy()
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('A uses', diff.total_seconds(), 'seconds')
scores = scores[:,
self._num_anchors['stride%s' % s]:, :, :]
idx += 1
bbox_deltas = net_out[idx].asnumpy()
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride, anchors_fpn)
anchors = anchors.reshape((K * A, 4))
scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
bbox_pred_len = bbox_deltas.shape[3] // A
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
scores_ravel = scores.ravel()
order = np.where(scores_ravel >= threshold)[0]
proposals = proposals[order, :]
scores = scores[order]
if stride == 4 and self.decay4 < 1.0:
scores *= self.decay4
if flip:
oldx1 = proposals[:, 0].copy()
oldx2 = proposals[:, 2].copy()
proposals[:, 0] = im.shape[1] - oldx2 - 1
proposals[:, 2] = im.shape[1] - oldx1 - 1
proposals[:, 0:4] /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
if not self.vote and self.use_landmarks:
idx += 1
landmark_deltas = net_out[idx].asnumpy()
landmark_deltas = self._clip_pad(
landmark_deltas, (height, width))
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose(
(0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
landmarks = self.landmark_pred(anchors,
landmark_deltas)
landmarks = landmarks[order, :]
if flip:
landmarks[:, :,
0] = im.shape[1] - landmarks[:, :, 0] - 1
order = [1, 0, 2, 4, 3]
flandmarks = landmarks.copy()
for idx, a in enumerate(order):
flandmarks[:, idx, :] = landmarks[:, a, :]
landmarks = flandmarks
landmarks[:, :, 0:2] /= im_scale
landmarks_list.append(landmarks)
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('B uses', diff.total_seconds(), 'seconds')
proposals = np.vstack(proposals_list)
landmarks = None
if proposals.shape[0] == 0:
if self.use_landmarks:
landmarks = np.zeros((0, 5, 2))
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
#print('shapes', proposals.shape, scores.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
#if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
proposals = proposals[order, :]
scores = scores[order]
if not self.vote and self.use_landmarks:
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if not self.vote:
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
if self.use_landmarks:
landmarks = landmarks[keep]
else:
det = np.hstack((pre_det, proposals[:, 4:]))
det = self.bbox_vote(det)
#if self.use_landmarks:
# det = np.hstack((det, landmarks))
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('C uses', diff.total_seconds(), 'seconds')
return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False):
#print('in_detect', threshold, scales, do_flip, do_nms)
proposals_list = []
scores_list = []
landmarks_list = []
strides_list = []
timea = datetime.datetime.now()
flips = [0]
if do_flip:
flips = [0, 1]
imgs = [img]
if isinstance(img, list):
imgs = img
for img in imgs:
for im_scale in scales:
for flip in flips:
if im_scale != 1.0:
im = cv2.resize(img,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
else:
im = img.copy()
if flip:
im = im[:, ::-1, :]
if self.nocrop:
if im.shape[0] % 32 == 0:
h = im.shape[0]
else:
h = (im.shape[0] // 32 + 1) * 32
if im.shape[1] % 32 == 0:
w = im.shape[1]
else:
w = (im.shape[1] // 32 + 1) * 32
_im = np.zeros((h, w, 3), dtype=np.float32)
_im[0:im.shape[0], 0:im.shape[1], :] = im
im = _im
else:
im = im.astype(np.float32)
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X1 uses', diff.total_seconds(), 'seconds')
#self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False)
#im_info = [im.shape[0], im.shape[1], im_scale]
im_info = [im.shape[0], im.shape[1]]
im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1]))
for i in range(3):
im_tensor[0, i, :, :] = (
im[:, :, 2 - i] / self.pixel_scale -
self.pixel_means[2 - i]) / self.pixel_stds[2 - i]
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X2 uses', diff.total_seconds(), 'seconds')
data = nd.array(im_tensor)
db = mx.io.DataBatch(data=(data, ),
provide_data=[('data', data.shape)])
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('X3 uses', diff.total_seconds(), 'seconds')
self.model.forward(db, is_train=False)
net_out = self.model.get_outputs()
#post_nms_topN = self._rpn_post_nms_top_n
#min_size_dict = self._rpn_min_size_fpn
sym_idx = 0
for _idx, s in enumerate(self._feat_stride_fpn):
#if len(scales)>1 and s==32 and im_scale==scales[-1]:
# continue
_key = 'stride%s' % s
stride = int(s)
is_cascade = False
if self.cascade:
is_cascade = True
#if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]):
# continue
#print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr)
scores = net_out[sym_idx].asnumpy()
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('A uses', diff.total_seconds(), 'seconds')
#print(scores.shape)
#print('scores',stride, scores.shape, file=sys.stderr)
scores = scores[:, self._num_anchors['stride%s' %
s]:, :, :]
bbox_deltas = net_out[sym_idx + 1].asnumpy()
#if DEBUG:
# print 'im_size: ({}, {})'.format(im_info[0], im_info[1])
# print 'scale: {}'.format(im_info[2])
#_height, _width = int(im_info[0] / stride), int(im_info[1] / stride)
height, width = bbox_deltas.shape[
2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride,
anchors_fpn)
#print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr)
anchors = anchors.reshape((K * A, 4))
#print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr)
#print('HW', (height, width), file=sys.stderr)
#print('anchors_fpn', anchors_fpn.shape, file=sys.stderr)
#print('anchors', anchors.shape, file=sys.stderr)
#print('bbox_deltas', bbox_deltas.shape, file=sys.stderr)
#print('scores', scores.shape, file=sys.stderr)
#scores = self._clip_pad(scores, (height, width))
scores = scores.transpose((0, 2, 3, 1)).reshape(
(-1, 1))
#print('pre', bbox_deltas.shape, height, width)
#bbox_deltas = self._clip_pad(bbox_deltas, (height, width))
#print('after', bbox_deltas.shape, height, width)
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
bbox_pred_len = bbox_deltas.shape[3] // A
#print(bbox_deltas.shape)
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
bbox_deltas[:,
0::4] = bbox_deltas[:, 0::
4] * self.bbox_stds[0]
bbox_deltas[:,
1::4] = bbox_deltas[:, 1::
4] * self.bbox_stds[1]
bbox_deltas[:,
2::4] = bbox_deltas[:, 2::
4] * self.bbox_stds[2]
bbox_deltas[:,
3::4] = bbox_deltas[:, 3::
4] * self.bbox_stds[3]
proposals = self.bbox_pred(anchors, bbox_deltas)
#print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr)
if is_cascade:
cascade_sym_num = 0
cls_cascade = False
bbox_cascade = False
__idx = [3, 4]
if not self.use_landmarks:
__idx = [2, 3]
for diff_idx in __idx:
if sym_idx + diff_idx >= len(net_out):
break
body = net_out[sym_idx + diff_idx].asnumpy()
if body.shape[1] // A == 2: #cls branch
if cls_cascade or bbox_cascade:
break
else:
cascade_scores = body[:, self.
_num_anchors[
'stride%s' %
s]:, :, :]
cascade_scores = cascade_scores.transpose(
(0, 2, 3, 1)).reshape((-1, 1))
#scores = (scores+cascade_scores)/2.0
scores = cascade_scores #TODO?
cascade_sym_num += 1
cls_cascade = True
#print('find cascade cls at stride', stride)
elif body.shape[1] // A == 4: #bbox branch
cascade_deltas = body.transpose(
(0, 2, 3, 1)).reshape(
(-1, bbox_pred_len))
cascade_deltas[:, 0::
4] = cascade_deltas[:, 0::
4] * self.bbox_stds[
0]
cascade_deltas[:, 1::
4] = cascade_deltas[:, 1::
4] * self.bbox_stds[
1]
cascade_deltas[:, 2::
4] = cascade_deltas[:, 2::
4] * self.bbox_stds[
2]
cascade_deltas[:, 3::
4] = cascade_deltas[:, 3::
4] * self.bbox_stds[
3]
proposals = self.bbox_pred(
proposals, cascade_deltas)
cascade_sym_num += 1
bbox_cascade = True
#print('find cascade bbox at stride', stride)
proposals = clip_boxes(proposals, im_info[:2])
#if self.vote:
# if im_scale>1.0:
# keep = self._filter_boxes2(proposals, 160*im_scale, -1)
# else:
# keep = self._filter_boxes2(proposals, -1, 100*im_scale)
# if stride==4:
# keep = self._filter_boxes2(proposals, 12*im_scale, -1)
# proposals = proposals[keep, :]
# scores = scores[keep]
#keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2])
#proposals = proposals[keep, :]
#scores = scores[keep]
#print('333', proposals.shape)
if stride == 4 and self.decay4 < 1.0:
scores *= self.decay4
scores_ravel = scores.ravel()
#print('__shapes', proposals.shape, scores_ravel.shape)
#print('max score', np.max(scores_ravel))
order = np.where(scores_ravel >= threshold)[0]
#_scores = scores_ravel[order]
#_order = _scores.argsort()[::-1]
#order = order[_order]
proposals = proposals[order, :]
scores = scores[order]
if flip:
oldx1 = proposals[:, 0].copy()
oldx2 = proposals[:, 2].copy()
proposals[:, 0] = im.shape[1] - oldx2 - 1
proposals[:, 2] = im.shape[1] - oldx1 - 1
proposals[:, 0:4] /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
if self.nms_threshold < 0.0:
_strides = np.empty(shape=(scores.shape),
dtype=np.float32)
_strides.fill(stride)
strides_list.append(_strides)
if not self.vote and self.use_landmarks:
landmark_deltas = net_out[sym_idx + 2].asnumpy()
#landmark_deltas = self._clip_pad(landmark_deltas, (height, width))
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose(
(0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
landmark_deltas *= self.landmark_std
#print(landmark_deltas.shape, landmark_deltas)
landmarks = self.landmark_pred(
anchors, landmark_deltas)
landmarks = landmarks[order, :]
if flip:
landmarks[:, :,
0] = im.shape[1] - landmarks[:, :,
0] - 1
#for a in range(5):
# oldx1 = landmarks[:, a].copy()
# landmarks[:,a] = im.shape[1] - oldx1 - 1
order = [1, 0, 2, 4, 3]
flandmarks = landmarks.copy()
for idx, a in enumerate(order):
flandmarks[:, idx, :] = landmarks[:, a, :]
#flandmarks[:, idx*2] = landmarks[:,a*2]
#flandmarks[:, idx*2+1] = landmarks[:,a*2+1]
landmarks = flandmarks
landmarks[:, :, 0:2] /= im_scale
#landmarks /= im_scale
#landmarks = landmarks.reshape( (-1, landmark_pred_len) )
landmarks_list.append(landmarks)
#proposals = np.hstack((proposals, landmarks))
if self.use_landmarks:
sym_idx += 3
else:
sym_idx += 2
if is_cascade:
sym_idx += cascade_sym_num
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('B uses', diff.total_seconds(), 'seconds')
proposals = np.vstack(proposals_list)
landmarks = None
if proposals.shape[0] == 0:
if self.use_landmarks:
landmarks = np.zeros((0, 5, 2))
if self.nms_threshold < 0.0:
return np.zeros((0, 6)), landmarks
else:
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
#print('shapes', proposals.shape, scores.shape)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
#if config.TEST.SCORE_THRESH>0.0:
# _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH)
# order = order[:_count]
proposals = proposals[order, :]
scores = scores[order]
if self.nms_threshold < 0.0:
strides = np.vstack(strides_list)
strides = strides[order]
if not self.vote and self.use_landmarks:
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
if self.nms_threshold > 0.0:
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if not self.vote:
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
if self.use_landmarks:
landmarks = landmarks[keep]
else:
det = np.hstack((pre_det, proposals[:, 4:]))
det = self.bbox_vote(det)
elif self.nms_threshold < 0.0:
det = np.hstack(
(proposals[:, 0:4], scores, strides)).astype(np.float32,
copy=False)
else:
det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
if self.debug:
timeb = datetime.datetime.now()
diff = timeb - timea
print('C uses', diff.total_seconds(), 'seconds')
return det, landmarks
def detect(self, img, threshold=0.5):
"""
Detect all the faces and landmarks in an image
:param img: input image
:param threshold: detection threshold
:return: tuple faces, landmarks
"""
proposals_list = []
scores_list = []
landmarks_list = []
im_shape = img.shape
target_size = self.scales[0]
max_size = self.scales[1]
im_size_min = np.min(im_shape[0:2])
im_size_max = np.max(im_shape[0:2])
im_scale = float(target_size) / float(im_size_min)
if np.round(im_scale * im_size_max) > max_size:
im_scale = float(max_size) / float(im_size_max)
if im_scale != 1.0:
img = cv2.resize(img,
None,
None,
fx=im_scale,
fy=im_scale,
interpolation=cv2.INTER_LINEAR)
img = img.astype(np.float32)
im_info = [img.shape[0], img.shape[1]]
im_tensor = np.zeros((1, 3, img.shape[0], img.shape[1]))
for i in range(3):
im_tensor[
0,
i, :, :] = (img[:, :, 2 - i] / self.pixel_scale -
self.pixel_means[2 - i]) / self.pixel_stds[2 - i]
net_out = self.model.predict(im_tensor.transpose(0, 2, 3, 1))
net_out_reshaped = []
for elt in net_out:
net_out_reshaped.append(elt.transpose(0, 3, 1, 2))
net_out = net_out_reshaped
sym_idx = 0
for _idx, s in enumerate(self._feat_stride_fpn):
_key = 'stride%s' % s
stride = int(s)
scores = net_out[sym_idx]
scores = scores[:, self._num_anchors['stride%s' % s]:, :, :]
bbox_deltas = net_out[sym_idx + 1]
height, width = bbox_deltas.shape[2], bbox_deltas.shape[3]
A = self._num_anchors['stride%s' % s]
K = height * width
anchors_fpn = self._anchors_fpn['stride%s' % s]
anchors = anchors_plane(height, width, stride, anchors_fpn)
anchors = anchors.reshape((K * A, 4))
scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1))
bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))
bbox_pred_len = bbox_deltas.shape[3] // A
bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len))
bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * self.bbox_stds[0]
bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * self.bbox_stds[1]
bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * self.bbox_stds[2]
bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * self.bbox_stds[3]
proposals = self.bbox_pred(anchors, bbox_deltas)
proposals = clip_boxes(proposals, im_info[:2])
if stride == 4 and self.decay4 < 1.0:
scores *= self.decay4
scores_ravel = scores.ravel()
order = np.where(scores_ravel >= threshold)[0]
proposals = proposals[order, :]
scores = scores[order]
proposals[:, 0:4] /= im_scale
proposals_list.append(proposals)
scores_list.append(scores)
landmark_deltas = net_out[sym_idx + 2]
landmark_pred_len = landmark_deltas.shape[1] // A
landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape(
(-1, 5, landmark_pred_len // 5))
landmarks = self.landmark_pred(anchors, landmark_deltas)
landmarks = landmarks[order, :]
landmarks[:, :, 0:2] /= im_scale
landmarks_list.append(landmarks)
sym_idx += 3
proposals = np.vstack(proposals_list)
if proposals.shape[0] == 0:
landmarks = np.zeros((0, 5, 2))
return np.zeros((0, 5)), landmarks
scores = np.vstack(scores_list)
scores_ravel = scores.ravel()
order = scores_ravel.argsort()[::-1]
proposals = proposals[order, :]
scores = scores[order]
landmarks = np.vstack(landmarks_list)
landmarks = landmarks[order].astype(np.float32, copy=False)
pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32,
copy=False)
keep = self.nms(pre_det)
det = np.hstack((pre_det, proposals[:, 4:]))
det = det[keep, :]
landmarks = landmarks[keep]
return det, landmarks
