def _retina_detach(self, net_out, index, stride): key = f'stride{stride}' A, F = self._num_anchors[key], self._anchors_fpn[key] [scores, deltas] = map(lambda x: x.asnumpy(), net_out[index:index+2]) height, width = deltas.shape[2], deltas.shape[3] scores = scores[:, A:, :, :].transpose((0, 2, 3, 1)).reshape((-1, 1)) deltas = deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) anchors = anchors_plane(height, width, stride, F).reshape((-1, 4)) return scores, deltas, anchors
def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0][0].context.device_id) #nms = cpu_nms_wrapper(self._threshold) cls_prob_dict = dict(zip(self.fpn_keys, in_data[0:len(self.fpn_keys)])) bbox_pred_dict = dict(zip(self.fpn_keys, in_data[len(self.fpn_keys):2*len(self.fpn_keys)])) #for i in xrange(6): # print(i, in_data[i].asnumpy().shape) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError("Sorry, multiple images each device is not implemented") pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size_dict = self._rpn_min_size_fpn proposals_list = [] scores_list = [] for s in self._feat_stride_fpn: stride = int(s) scores = cls_prob_dict['stride%s'%s].asnumpy()[:, self._num_anchors['stride%s'%s]:, :, :] bbox_deltas = bbox_pred_dict['stride%s'%s].asnumpy() im_info = in_data[-1].asnumpy()[0, :] if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) #height, width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s'%s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32)) anchors = anchors.reshape((K * A, 4)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K) proposals = self._bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) if np.shape(det)[0] == 0: print "Something wrong with the input image(resolution is too low?), generate fake proposals for it." proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) keep = nms(det) if post_nms_topN > 0: keep = keep[:post_nms_topN] if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def detect(self, img, threshold=0.5, im_scale=1.0): proposals_list = [] scores_list = [] landmarks_list = [] data = nd.array(img) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s stride = int(s) if self.use_landmarks: idx = _idx * 3 else: idx = _idx * 2 scores = net_out[idx].asnumpy() scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, (img.shape[2], img.shape[3])) scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): #print('in_detect', threshold, scales, do_flip, do_nms) #print('img_shape: ',img.shape) proposals_list = [] scores_list = [] landmarks_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() im = im[:, :, np.newaxis] if flip: im = im[:, ::-1, :] if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 1), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X1 uses', diff.total_seconds(), 'seconds') #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) #im_info = [im.shape[0], im.shape[1], im_scale] im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 1, im.shape[0], im.shape[1])) #print('im_tensor_shape: ', im_tensor.shape) #print('im_shape: ', im.shape) for i in range(1): im_tensor[ 0, i, :, :] = (im[:, :, i] / self.pixel_scale - self.pixel_means[i]) / self.pixel_stds[i] if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X2 uses', diff.total_seconds(), 'seconds') data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X3 uses', diff.total_seconds(), 'seconds') self.model.forward(db, is_train=False) net_out = self.model.get_outputs() #print('Len:out: ', len(net_out)) #print('out1: ', net_out[0]) #print('outbbox: ', net_out[1]) #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn for _idx, s in enumerate(self._feat_stride_fpn): #if len(scales)>1 and s==32 and im_scale==scales[-1]: # continue _key = 'stride%s' % s stride = int(s) #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): # continue if self.use_landmarks: idx = _idx * 3 else: idx = _idx * 2 #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('A uses', diff.total_seconds(), 'seconds') #print(scores.shape) #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] #print('A: ',A) K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) #print('HW', (height, width), file=sys.stderr) #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) #print('anchors', anchors.shape, file=sys.stderr) #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) #print('scores', scores.shape, file=sys.stderr) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) #print('bbox_deltas.shape[3]:',bbox_deltas.shape[3]) bbox_pred_len = bbox_deltas.shape[3] // A #print('bbox_deltas.shape:',bbox_deltas.shape) #print('boxlen:',bbox_pred_len) bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) #if self.vote: # if im_scale>1.0: # keep = self._filter_boxes2(proposals, 160*im_scale, -1) # else: # keep = self._filter_boxes2(proposals, -1, 100*im_scale) # if stride==4: # keep = self._filter_boxes2(proposals, 12*im_scale, -1) # proposals = proposals[keep, :] # scores = scores[keep] #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() #print('__shapes', proposals.shape, scores_ravel.shape) #print('max score', np.max(scores_ravel)) order = np.where(scores_ravel >= threshold)[0] #print('order:',order) #print('score_order:',scores_ravel[order]) #_scores = scores_ravel[order] #_order = _scores.argsort()[::-1] #order = order[_order] #order = [50] proposals = proposals[order, :] scores = scores[order] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = self._clip_pad( landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) #print(landmark_deltas.shape, landmark_deltas) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 #for a in range(5): # oldx1 = landmarks[:, a].copy() # landmarks[:,a] = im.shape[1] - oldx1 - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] #flandmarks[:, idx*2] = landmarks[:,a*2] #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale #landmarks /= im_scale #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) landmarks_list.append(landmarks) #proposals = np.hstack((proposals, landmarks)) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('B uses', diff.total_seconds(), 'seconds') proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) #print('shapes', proposals.shape, scores.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] proposals = proposals[order, :] scores = scores[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: #print('pre_det_type: ', type(pre_det)) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) #if self.use_landmarks: # det = np.hstack((det, landmarks)) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('C uses', diff.total_seconds(), 'seconds') return det, scores #landmarks
def detect(self, img, scales_index=0): proposals_list = [] scores_list = [] im_src = img.copy() CONSTANT = config.TEST.CONSTANT BLACK = [0, 0, 0] img = cv2.copyMakeBorder(img, CONSTANT, CONSTANT, CONSTANT, CONSTANT, cv2.BORDER_CONSTANT, value=BLACK) scales = self.get_boxes(img, scales_index) for im_scale in scales: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) # self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n for s in self._feat_stride_fpn: if len(scales) > 1 and s == 32 and im_scale == scales[-1]: continue _key = 'stride%s' % s stride = int(s) idx = 0 if s == 16: idx = 2 elif s == 8: idx = 4 # print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() # print(scores.shape) idx += 1 # print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[idx].asnumpy() _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) # print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) # print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) # print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) # proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals /= im_scale # #add by sai with pyramidbox to filt scale face # if im_scale > 1: # index = np.where( # np.minimum(proposals[:, 2] - proposals[:, 0] + 1, # proposals[:, 3] - proposals[:, 1] + 1) < 50)[0] # proposals = proposals[index, :] # scores = scores[index, :] # else: # index = np.where( # np.maximum(proposals[:, 2] - proposals[:, 0] + 1, # proposals[:, 3] - proposals[:, 1] + 1) > 20)[0] # proposals = proposals[index, :] # scores = scores[index, :] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) if self.nms_threshold < 1.0: keep = self.nms(det) det = det[keep, :] threshold = config.TEST.SCORE_THRESH if threshold > 0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] # add by sai if det.shape[0] != 0: for i in range(det.shape[0]): det[i, :][0] = det[i, :][0] - CONSTANT det[i, :][1] = det[i, :][1] - CONSTANT det[i, :][2] = det[i, :][2] - CONSTANT det[i, :][3] = det[i, :][3] - CONSTANT if det[i, :][0] < 0: det[i, :][0] = 0 if det[i, :][2] > im_src.shape[1]: det[i, :][2] = im_src.shape[1] if det[i, :][1] < 0: det[i, :][1] = 0 if det[i, :][3] > im_src.shape[0]: det[i, :][3] = im_src.shape[0] return det
def postprocess(net_out, threshold, ctx_id, im_scale, im_info): # im_info = [640, 640] flip = False decay4 = 0.5 vote = False fpn_keys = [] anchor_cfg = None bbox_stds = [1.0, 1.0, 1.0, 1.0] # im_scale = 1.0 landmark_std = 1.0 nms_threshold = 0.4 proposals_list = [] scores_list = [] landmarks_list = [] strides_list = [] use_landmarks = True if ctx_id >= 0: nms = gpu_nms_wrapper(nms_threshold, ctx_id) else: nms = cpu_nms_wrapper(nms_threshold) use_landmarks = True _ratio = (1., ) _feat_stride_fpn = [32, 16, 8] anchor_cfg = { '32': { 'SCALES': (32, 16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999 }, '16': { 'SCALES': (8, 4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999 }, '8': { 'SCALES': (2, 1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999 }, } for s in _feat_stride_fpn: fpn_keys.append('stride%s' % s) dense_anchor = False _anchors_fpn = dict( zip(fpn_keys, generate_anchors_fpn(dense_anchor=dense_anchor, cfg=anchor_cfg))) for k in _anchors_fpn: v = _anchors_fpn[k].astype(np.float32) _anchors_fpn[k] = v _num_anchors = dict( zip(fpn_keys, [anchors.shape[0] for anchors in _anchors_fpn.values()])) sym_idx = 0 for _idx, s in enumerate(_feat_stride_fpn): # print(sym_idx) _key = 'stride%s' % s # print(_key) stride = int(s) scores = net_out[sym_idx] #.asnumpy() scores = scores[:, _num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[sym_idx + 1] # .asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = _num_anchors['stride%s' % s] K = height * width anchors_fpn = _anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * bbox_stds[3] proposals = bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) if stride == 4 and decay4 < 1.0: scores *= decay4 scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 #proposals[:,0:4] /= im_scale #print(proposals[:,0]) proposals[:, 0] /= im_scale[0] #print(pp) proposals[:, 1] /= im_scale[1] proposals[:, 2] /= im_scale[0] proposals[:, 3] /= im_scale[1] #print(proposals[:,0]) proposals_list.append(proposals) scores_list.append(scores) if nms_threshold < 0.0: _strides = np.empty(shape=(scores.shape), dtype=np.float32) _strides.fill(stride) strides_list.append(_strides) if not vote and use_landmarks: landmark_deltas = net_out[sym_idx + 2] #.asnumpy() # print(landmark_deltas) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmark_deltas *= landmark_std landmarks = landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) if use_landmarks: sym_idx += 3 else: sym_idx += 2 proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if use_landmarks: landmarks = np.zeros((0, 5, 2)) if nms_threshold < 0.0: return np.zeros((0, 6)), landmarks else: return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] if nms_threshold < 0.0: strides = np.vstack(strides_list) strides = strides[order] if not vote and use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) if nms_threshold > 0.0: pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not vote: keep = nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = bbox_vote(det, nms_threshold) elif nms_threshold < 0.0: det = np.hstack((proposals[:, 0:4], scores, strides)).astype(np.float32, copy=False) else: det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) return det, landmarks
def detect(self, img, threshold=0.5): """ Detect all the faces and landmarks in an image :param img: input image :param threshold: detection threshold :return: tuple faces, landmarks """ proposals_list = [] scores_list = [] landmarks_list = [] im_tensor, im_info, im_scale = self._preprocess_image(img) net_out = self.model(im_tensor) net_out = [elt.numpy() for elt in net_out] sym_idx = 0 for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s scores = net_out[sym_idx] scores = scores[:, :, :, self._num_anchors['stride%s' % s]:] bbox_deltas = net_out[sym_idx + 1] height, width = bbox_deltas.shape[1], bbox_deltas.shape[2] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, s, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = scores.reshape((-1, 1)) bbox_deltas = bbox_deltas bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * self.bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * self.bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * self.bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * self.bbox_stds[3] proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) if s == 4 and self.decay4 < 1.0: scores *= self.decay4 scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) landmark_deltas = net_out[sym_idx + 2] landmark_pred_len = landmark_deltas.shape[3] // A landmark_deltas = landmark_deltas.reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) sym_idx += 3 proposals = np.vstack(proposals_list) if proposals.shape[0] == 0: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] landmarks = landmarks[keep] return det, landmarks
def detect(self,img, threshold=0.05): image_size = (img.shape[0], img.shape[1]) #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [image_size[0], image_size[1], 1.0] data = nd.zeros( (1 ,3, image_size[0], image_size[1]) ) nimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) nimg = np.transpose(nimg, (2,0,1)) nimg = nd.array(nimg) data[0] = nimg db = mx.io.DataBatch(data=(data,)) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn proposals_list = [] scores_list = [] idx = 0 for s in self._feat_stride_fpn: _key = 'stride%s'%s stride = int(s) scores = net_out[idx].asnumpy() #print(scores.shape) idx+=1 #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s'%s]:, :, :] bbox_deltas = net_out[idx].asnumpy() idx+=1 #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s'%s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32)) print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) #proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] #if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) #if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self.nms_threshold<1.0: keep = self.nms(det) det = det[keep, :] if threshold>0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] return det
def detect(self, img, threshold=0.5): proposals_list = [] scores_list = [] landmarks_list = [] im_info = [640, 640] if img.shape[0] != img.shape[1]: BLUE = (255, 0, 0) if img.shape[0] > img.shape[1]: img = cv2.copyMakeBorder(img, 0, 0, 0, img.shape[0] - img.shape[1], cv2.BORDER_CONSTANT, value=BLUE) else: img = cv2.copyMakeBorder(img, 0, img.shape[1] - img.shape[0], 0, 0, cv2.BORDER_CONSTANT, value=BLUE) re_scale = float(im_info[0]) / float(img.shape[0]) img = cv2.resize(img, (im_info[0], im_info[1])) img = img.astype(np.float32) im_tensor = np.zeros((1, 3, img.shape[0], img.shape[1])) for i in range(3): im_tensor[ 0, i, :, :] = (img[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() for _idx, s in enumerate(self._feat_stride_fpn): stride = int(s) idx = _idx * 3 scores = net_out[idx].asnumpy() scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() idx += 1 landmark_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] scores = scores[order] proposals = proposals[order, :] landmarks = landmarks[order, :] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 proposals[:, 0:4] /= re_scale landmarks[:, :, 0:2] /= re_scale scores_list.append(scores) proposals_list.append(proposals) landmarks_list.append(landmarks) scores = np.vstack(scores_list) proposals = np.vstack(proposals_list) landmarks = np.vstack(landmarks_list) if proposals.shape[0] == 0: return np.zeros((0, 5)), np.zeros((0, 5, 2)) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] scores = scores[order] proposals = proposals[order, :] landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] landmarks = landmarks[keep] return det, landmarks
def detect(self, img, threshold=0.05, scales=[1.0]): proposals_list = [] scores_list = [] for im_scale in scales: if im_scale!=1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn for s in self._feat_stride_fpn: if len(scales)>1 and s==32 and im_scale==scales[-1]: continue _key = 'stride%s'%s stride = int(s) idx = 0 if s==16: idx=2 elif s==8: idx=4 print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() #print(scores.shape) idx+=1 #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s'%s]:, :, :] bbox_deltas = net_out[idx].asnumpy() #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s'%s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32)) #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) #proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals /= im_scale proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] #if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) #if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self.nms_threshold<1.0: keep = self.nms(det) det = det[keep, :] if threshold>0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] return det
def detect(self, img, thresh, scales=[1.0], do_flip=False): proposal_list = [] scores_list = [] landmarks_list = [] flips = [0,1] if do_flip else [0] for im_scale in scales: for flip in flips: if im_scale!=1.0: img = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: img = img.copy() if flip: img = img[:,::-1,:] # img = self.pad_img_to_32(img) imgshape = [img.shape[0], img.shape[1]] img = self.preprocess(img) img = torch.from_numpy(img) if self.use_gpu: img = img.cuda() net_out = self.model(img) for _idx,s in enumerate(self.fpn_keys): idx = _idx * 3 scores = net_out[idx].detach().cpu().numpy() scores = scores[:, self._num_anchors[s]:] idx += 1 bbox_deltas = net_out[idx].detach().cpu().numpy() h, w = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors[s] K = h*w anchors_fpn = self._anchors_fpn[s] anchors_fpn = np.float32(anchors_fpn) anchors = anchors_plane(h, w, s, anchors_fpn) anchors = anchors.reshape((K*A, 4)) scores = self._clip_pad(scores, (h, w)) scores = scores.transpose([0,2,3,1]).reshape([-1,1]) # print('SCR') # print(scores) # print(scores.shape) # input() bbox_deltas = self._clip_pad(bbox_deltas, (h,w)) bbox_deltas = bbox_deltas.transpose([0,2,3,1]) bbox_pred_len = bbox_deltas.shape[3]//A bbox_deltas = bbox_deltas.reshape([-1, bbox_pred_len]) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, imgshape) scores_ravel = scores.ravel() order = np.where(scores_ravel>=thresh)[0] proposals = proposals[order] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:,:4] /= im_scale # print('proposals') # print(proposals) # print(proposals.shape) # input() proposal_list.append(proposals) scores_list.append(scores) # landmarks idx += 1 landmark_deltas = net_out[idx].detach().cpu().numpy() landmark_deltas = self._clip_pad(landmark_deltas, (h,w)) landmark_pred_len = landmark_deltas.shape[1]//A landmark_deltas = landmark_deltas.transpose((0,2,3,1)).reshape([-1,5,landmark_pred_len//5]) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:,:,0] = imgshape[1] - landmarks[:,:,0] - 1 order = [1,0,2,4,3] flandmarks = landmarks[:,np.int32(order)] landmarks[:,:,:2] /= im_scale landmarks_list.append(landmarks) # print('PROPOSAL', proposal_list) proposals = np.vstack(proposal_list) landmarks = None if proposals.shape[0]==0: return np.zeros([0,5]), np.zeros([0,5,2]) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order] scores = scores[order] landmarks = np.vstack(landmarks_list) landmarks = np.float32(landmarks[order]) pre_det = np.hstack([proposals[:, 0:4], scores]) pre_det = np.float32(pre_det) keep = self.nms(pre_det) det = np.hstack([pre_det, proposals[:,4:]]) det = det[keep] landmarks = landmarks[keep] return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): print( 'get into detect, confi thresold={}, scales={}, do_flip={}'.format( threshold, scales, do_flip)) proposals_list = [] scores_list = [] landmarks_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] #TODO 根据scale给输入的图片做resize for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() # 对图像做翻转 if flip: im = im[:, ::-1, :] # 对图像做裁剪 if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 3), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) im_info = [im.shape[0], im.shape[1]] #h,w im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = ( im[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[ 2 - i] #TODO 这里好像将Channel顺序倒过来了,与image.py保持一致 data = np.array(im_tensor) # 读入模型进行推理,得到预测值 net_out = self.get_pred(data) if self.debug: for key in net_out.keys(): print('{} = {}\n'.format(key, net_out[key].shape)) for _idx, s in enumerate(self._feat_stride_fpn): # print('begin stride{}-------------------------------------------------\n'.format(s)) _key = 'stride%s' % s stride = int(s) # print('getting im_scale={}, stride={}, len(net_out)={}, data.shape={}'.format(im_scale, stride, len(net_out), data.shape)) scores = net_out['rpn_cls_prob_stride%s' % s] #TODO 要注意这里是nhwc不是nchw if self.debug: print('get score:', scores.shape) # print('stride{}: scores before shape={}, idx={}'.format(stride, scores.shape, self._num_anchors['stride%s' % s])) scores = scores[:, 1].reshape( (-1, 1)) #TODO: (H*W*A, 1) #这里的1表示正类的概率 if self.debug: print('AAAAstride{}: scores after shape={}'.format( stride, scores.shape)) bbox_deltas = net_out['rpn_bbox_pred_stride%s' % s] #TODO NHW8 height, width = bbox_deltas.shape[1], bbox_deltas.shape[2] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) #获取该特征图上的所有anchor #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) if self.debug: print('HW', (height, width)) print('anchors_fpn', anchors_fpn) print('anchors', anchors.shape, '\n') # scores = self._clip_pad_NCHW(scores, (height, width)) #(1, 4, H, W) # scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #(1, H, W, 4) # print('scores reshape', scores.shape) if self.debug: print('before bbox_deltas', bbox_deltas.shape) bbox_deltas = self._clip_pad_NHWC( bbox_deltas, (height, width)) #(1, H, W, 8) # bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))#(1, H, W, 8) bbox_pred_len = bbox_deltas.shape[3] // A #4 bbox_deltas = bbox_deltas.reshape( (-1, bbox_pred_len)) #(H*W*2, 4) if self.debug: print('after bbox_deltas', bbox_deltas.shape, height, width, '\n') #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self.bbox_pred( anchors, bbox_deltas) #TODO important! 将anchor加上delta进行处理 proposals = clip_boxes(proposals, im_info[:2]) scores_ravel = scores.ravel() max_score = np.max(scores_ravel) print('proposals.shape={}, score_ravel.shape={}'.format( proposals.shape, scores_ravel.shape)) print('max score', max_score) order = np.where(scores_ravel >= threshold)[0] #_scores = scores_ravel[order] #_order = _scores.argsort()[::-1] #order = order[_order] proposals = proposals[order, :] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0: 4] /= im_scale #TODO important 在这里将找到的proposal给映射回原来图像的位置 proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: landmark_deltas = net_out['rpn_landmark_pred_stride%s' % s] #(1,20,H,W) if self.debug: print('before landmark_deltas', landmark_deltas.shape) landmark_deltas = self._clip_pad_NCHW( landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) if self.debug: print('after landmark_deltas', landmark_deltas.shape, landmark_deltas) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 #for a in range(5): # oldx1 = landmarks[:, a].copy() # landmarks[:,a] = im.shape[1] - oldx1 - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] #flandmarks[:, idx*2] = landmarks[:,a*2] #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) if self.debug: print( 'end stride{}-------------------------------------------------\n' .format(s)) proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks # for i in range(len(scores_list)): # print('hhhhh score,shape=',scores_list[i].shape) scores = np.vstack(scores_list) print('finally!!! proposals.shape={}, score.shape={}'.format( proposals.shape, scores.shape)) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] # 按照score从大到小排序 proposals = proposals[order, :] scores = scores[order] if self.debug: print('sort score=', scores) if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: print('begin to NMS!!\n') keep = self.nms(pre_det) # print('before hstack: pre_det={}, proposals.shape={}, proposals[:,4:]={}'.format(pre_det.shape, proposals.shape, proposals[:,4:])) det = np.hstack((pre_det, proposals[:, 4:])) # print('after hstack: pre_det={}, proposals.shape={}'.format(pre_det.shape, proposals.shape)) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) return det, landmarks
def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0][0].context.device_id) cls_prob_dict = dict(zip(self.fpn_keys, in_data[0:len(self.fpn_keys)])) bbox_pred_dict = dict( zip(self.fpn_keys, in_data[len(self.fpn_keys):2 * len(self.fpn_keys)])) #for i in xrange(6): # print(i, in_data[i].asnumpy().shape) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError( "Sorry, multiple images each device is not implemented") pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size_dict = self._rpn_min_size_fpn proposals_list = [] scores_list = [] for s in self._feat_stride_fpn: _key = 'stride%s' % s stride = int(s) scores = cls_prob_dict[_key].asnumpy() #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = bbox_pred_dict['stride%s' % s].asnumpy() im_info = in_data[-1].asnumpy()[0, :] #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane( height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) #proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] #if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) #if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self._threshold < 1.0: keep = nms(det) else: keep = range(det.shape[0]) #print(det.shape, len(keep), post_nms_topN) if post_nms_topN > 0: keep = keep[:post_nms_topN] #print(det.shape, len(keep), post_nms_topN) num_keep = len(keep) #print('keep', keep, file=sys.stderr) if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] scores[num_keep:, :] = -1.0 #print('333 proposals', proposals[0:5,:], file=sys.stderr) #print('det', det.shape, num_keep) #print('first proposal', proposals[0], file=sys.stderr) batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def detect(self, img, threshold=0.05, scales=[1.0]): proposals_list = [] scores_list = [] for im_scale in scales: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) # self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n # post_nms_topN = self._rpn_post_nms_top_n # min_size_dict = self._rpn_min_size_fpn for s in self._feat_stride_fpn: if len(scales) > 1 and s == 32 and im_scale == scales[-1]: continue _key = 'stride%s' % s stride = int(s) idx = 0 if s == 16: idx = 2 elif s == 8: idx = 4 print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() # print(scores.shape) idx += 1 # print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[idx].asnumpy() # if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane( height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) # print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) # print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) # print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) # proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) # keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) # proposals = proposals[keep, :] # scores = scores[keep] # print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals /= im_scale proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] # if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] # if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) # if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self.nms_threshold < 1.0: keep = self.nms(det) det = det[keep, :] if threshold > 0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] return det
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): proposals_list = [] scores_list = [] landmarks_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() if flip: im = im[:, ::-1, :] if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 3), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X1 uses', diff.total_seconds(), 'seconds') im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = ( im[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X2 uses', diff.total_seconds(), 'seconds') data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X3 uses', diff.total_seconds(), 'seconds') self.model.forward(db, is_train=False) net_out = self.model.get_outputs() for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s stride = int(s) if self.use_landmarks: idx = _idx * 3 else: idx = _idx * 2 scores = net_out[idx].asnumpy() if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('A uses', diff.total_seconds(), 'seconds') scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = self._clip_pad( landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('B uses', diff.total_seconds(), 'seconds') proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) #print('shapes', proposals.shape, scores.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] proposals = proposals[order, :] scores = scores[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) #if self.use_landmarks: # det = np.hstack((det, landmarks)) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('C uses', diff.total_seconds(), 'seconds') return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): #print('in_detect', threshold, scales, do_flip, do_nms) proposals_list = [] scores_list = [] landmarks_list = [] strides_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] imgs = [img] if isinstance(img, list): imgs = img for img in imgs: for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() if flip: im = im[:, ::-1, :] if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 3), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X1 uses', diff.total_seconds(), 'seconds') #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) #im_info = [im.shape[0], im.shape[1], im_scale] im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = ( im[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X2 uses', diff.total_seconds(), 'seconds') data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X3 uses', diff.total_seconds(), 'seconds') self.model.forward(db, is_train=False) net_out = self.model.get_outputs() #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn sym_idx = 0 for _idx, s in enumerate(self._feat_stride_fpn): #if len(scales)>1 and s==32 and im_scale==scales[-1]: # continue _key = 'stride%s' % s stride = int(s) is_cascade = False if self.cascade: is_cascade = True #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): # continue #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[sym_idx].asnumpy() if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('A uses', diff.total_seconds(), 'seconds') #print(scores.shape) #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[sym_idx + 1].asnumpy() #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[ 2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) #print('HW', (height, width), file=sys.stderr) #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) #print('anchors', anchors.shape, file=sys.stderr) #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) #print('scores', scores.shape, file=sys.stderr) #scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape( (-1, 1)) #print('pre', bbox_deltas.shape, height, width) #bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A #print(bbox_deltas.shape) bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0:: 4] * self.bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1:: 4] * self.bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2:: 4] * self.bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3:: 4] * self.bbox_stds[3] proposals = self.bbox_pred(anchors, bbox_deltas) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) if is_cascade: cascade_sym_num = 0 cls_cascade = False bbox_cascade = False __idx = [3, 4] if not self.use_landmarks: __idx = [2, 3] for diff_idx in __idx: if sym_idx + diff_idx >= len(net_out): break body = net_out[sym_idx + diff_idx].asnumpy() if body.shape[1] // A == 2: #cls branch if cls_cascade or bbox_cascade: break else: cascade_scores = body[:, self. _num_anchors[ 'stride%s' % s]:, :, :] cascade_scores = cascade_scores.transpose( (0, 2, 3, 1)).reshape((-1, 1)) #scores = (scores+cascade_scores)/2.0 scores = cascade_scores #TODO? cascade_sym_num += 1 cls_cascade = True #print('find cascade cls at stride', stride) elif body.shape[1] // A == 4: #bbox branch cascade_deltas = body.transpose( (0, 2, 3, 1)).reshape( (-1, bbox_pred_len)) cascade_deltas[:, 0:: 4] = cascade_deltas[:, 0:: 4] * self.bbox_stds[ 0] cascade_deltas[:, 1:: 4] = cascade_deltas[:, 1:: 4] * self.bbox_stds[ 1] cascade_deltas[:, 2:: 4] = cascade_deltas[:, 2:: 4] * self.bbox_stds[ 2] cascade_deltas[:, 3:: 4] = cascade_deltas[:, 3:: 4] * self.bbox_stds[ 3] proposals = self.bbox_pred( proposals, cascade_deltas) cascade_sym_num += 1 bbox_cascade = True #print('find cascade bbox at stride', stride) proposals = clip_boxes(proposals, im_info[:2]) #if self.vote: # if im_scale>1.0: # keep = self._filter_boxes2(proposals, 160*im_scale, -1) # else: # keep = self._filter_boxes2(proposals, -1, 100*im_scale) # if stride==4: # keep = self._filter_boxes2(proposals, 12*im_scale, -1) # proposals = proposals[keep, :] # scores = scores[keep] #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 scores_ravel = scores.ravel() #print('__shapes', proposals.shape, scores_ravel.shape) #print('max score', np.max(scores_ravel)) order = np.where(scores_ravel >= threshold)[0] #_scores = scores_ravel[order] #_order = _scores.argsort()[::-1] #order = order[_order] proposals = proposals[order, :] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if self.nms_threshold < 0.0: _strides = np.empty(shape=(scores.shape), dtype=np.float32) _strides.fill(stride) strides_list.append(_strides) if not self.vote and self.use_landmarks: landmark_deltas = net_out[sym_idx + 2].asnumpy() #landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmark_deltas *= self.landmark_std #print(landmark_deltas.shape, landmark_deltas) landmarks = self.landmark_pred( anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 #for a in range(5): # oldx1 = landmarks[:, a].copy() # landmarks[:,a] = im.shape[1] - oldx1 - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] #flandmarks[:, idx*2] = landmarks[:,a*2] #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale #landmarks /= im_scale #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) landmarks_list.append(landmarks) #proposals = np.hstack((proposals, landmarks)) if self.use_landmarks: sym_idx += 3 else: sym_idx += 2 if is_cascade: sym_idx += cascade_sym_num if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('B uses', diff.total_seconds(), 'seconds') proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) if self.nms_threshold < 0.0: return np.zeros((0, 6)), landmarks else: return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) #print('shapes', proposals.shape, scores.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] proposals = proposals[order, :] scores = scores[order] if self.nms_threshold < 0.0: strides = np.vstack(strides_list) strides = strides[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) if self.nms_threshold > 0.0: pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) elif self.nms_threshold < 0.0: det = np.hstack( (proposals[:, 0:4], scores, strides)).astype(np.float32, copy=False) else: det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('C uses', diff.total_seconds(), 'seconds') return det, landmarks
def detect(self, img, threshold=0.5): """ Detect all the faces and landmarks in an image :param img: input image :param threshold: detection threshold :return: tuple faces, landmarks """ proposals_list = [] scores_list = [] landmarks_list = [] im_shape = img.shape target_size = self.scales[0] max_size = self.scales[1] im_size_min = np.min(im_shape[0:2]) im_size_max = np.max(im_shape[0:2]) im_scale = float(target_size) / float(im_size_min) if np.round(im_scale * im_size_max) > max_size: im_scale = float(max_size) / float(im_size_max) if im_scale != 1.0: img = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) img = img.astype(np.float32) im_info = [img.shape[0], img.shape[1]] im_tensor = np.zeros((1, 3, img.shape[0], img.shape[1])) for i in range(3): im_tensor[ 0, i, :, :] = (img[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] net_out = self.model.predict(im_tensor.transpose(0, 2, 3, 1)) net_out_reshaped = [] for elt in net_out: net_out_reshaped.append(elt.transpose(0, 3, 1, 2)) net_out = net_out_reshaped sym_idx = 0 for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s stride = int(s) scores = net_out[sym_idx] scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[sym_idx + 1] height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * self.bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * self.bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * self.bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * self.bbox_stds[3] proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) landmark_deltas = net_out[sym_idx + 2] landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) sym_idx += 3 proposals = np.vstack(proposals_list) if proposals.shape[0] == 0: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] landmarks = landmarks[keep] return det, landmarks