def im_detect(self, im_array, roi_array): """ perform detection of designated im, box, must follow minibatch.get_testbatch format :param im_array: numpy.ndarray [b c h w] :param roi_array: numpy.ndarray [roi_num 5] :return: scores, pred_boxes """ # remove duplicate feature rois if config.TEST.DEDUP_BOXES > 0: roi_array = roi_array # rank roi by v .* (b, dx, dy, dw, dh) v = np.array([1, 1e3, 1e6, 1e9, 1e12]) # create hash and inverse index for rois hashes = np.round(roi_array * config.TEST.DEDUP_BOXES).dot(v) _, index, inv_index = np.unique(hashes, return_index=True, return_inverse=True) roi_array = roi_array[index, :] self.arg_params['data'] = mx.nd.array(im_array, self.ctx) self.arg_params['rois'] = mx.nd.array(roi_array, self.ctx) arg_shapes, out_shapes, aux_shapes = \ self.symbol.infer_shape(data=self.arg_params['data'].shape, rois=self.arg_params['rois'].shape) arg_shapes_dict = { name: shape for name, shape in zip(self.symbol.list_arguments(), arg_shapes) } self.arg_params['cls_prob_label'] = mx.nd.zeros( arg_shapes_dict['cls_prob_label'], self.ctx) aux_names = self.symbol.list_auxiliary_states() self.aux_params = { k: mx.nd.zeros(s, self.ctx) for k, s in zip(aux_names, aux_shapes) } self.executor = self.symbol.bind(self.ctx, self.arg_params, args_grad=None, grad_req='null', aux_states=self.aux_params) output_dict = { name: nd for name, nd in zip(self.symbol.list_outputs(), self.executor.outputs) } self.executor.forward(is_train=False) scores = output_dict['cls_prob_output'].asnumpy() bbox_deltas = output_dict['bbox_pred_output'].asnumpy() pred_boxes = bbox_pred(roi_array[:, 1:], bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_array[0].shape[-2:]) if config.TEST.DEDUP_BOXES > 0: # map back to original scores = scores[inv_index, :] pred_boxes = pred_boxes[inv_index, :] return scores, pred_boxes
def main(): color = cv2.imread(args.img) # read image in b,g,r order img, scale = resize(color.copy(), 640, 1024) im_info = np.array([[img.shape[0], img.shape[1], scale]], dtype=np.float32) # (h, w, scale) img = np.swapaxes(img, 0, 2) img = np.swapaxes(img, 1, 2) # change to r,g,b order img = img[np.newaxis, :] # extend to (n, c, h, w) ctx = mx.gpu(args.gpu) _, arg_params, aux_params = mx.model.load_checkpoint( args.prefix, args.epoch) arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) if 'resnet' in args.prefix: sym = resnet_50(num_class=2, bn_mom=0.99, bn_global=True, is_train=False) else: sym = get_vgg_test(num_classes=2) arg_params["data"] = mx.nd.array(img, ctx) arg_params["im_info"] = mx.nd.array(im_info, ctx) exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params) exe.forward(is_train=False) output_dict = { name: nd for name, nd in zip(sym.list_outputs(), exe.outputs) } rois = output_dict['rpn_rois_output'].asnumpy( )[:, 1:] # first column is index scores = output_dict['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0] pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1])) cls_boxes = pred_boxes[:, 4:8] cls_scores = scores[:, 1] keep = np.where(cls_scores >= args.thresh)[0] cls_boxes = cls_boxes[keep, :] cls_scores = cls_scores[keep] dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32) keep = nms(dets.astype(np.float32), args.nms_thresh) dets = dets[keep, :] keep = nest(dets, thresh=args.nest_thresh) dets = dets[keep, :] for i in range(dets.shape[0]): bbox = dets[i, :4] cv2.rectangle( color, (int(round(bbox[0] / scale)), int(round(bbox[1] / scale))), (int(round(bbox[2] / scale)), int(round(bbox[3] / scale))), (0, 255, 0), 2) cv2.imwrite("result.jpg", color)
def im_detect(self, im_array, roi_array): """ perform detection of designated im, box, must follow minibatch.get_testbatch format :param im_array: numpy.ndarray [b c h w] :param roi_array: numpy.ndarray [roi_num 5] :return: scores, pred_boxes """ # remove duplicate feature rois if config.TEST.DEDUP_BOXES > 0: roi_array = roi_array # rank roi by v .* (b, dx, dy, dw, dh) v = np.array([1, 1e3, 1e6, 1e9, 1e12]) # create hash and inverse index for rois hashes = np.round(roi_array * config.TEST.DEDUP_BOXES).dot(v) _, index, inv_index = np.unique(hashes, return_index=True, return_inverse=True) roi_array = roi_array[index, :] self.arg_params['data'] = mx.nd.array(im_array, self.ctx) self.arg_params['rois'] = mx.nd.array(roi_array, self.ctx) arg_shapes, out_shapes, aux_shapes = \ self.symbol.infer_shape(data=self.arg_params['data'].shape, rois=self.arg_params['rois'].shape) arg_shapes_dict = {name: shape for name, shape in zip(self.symbol.list_arguments(), arg_shapes)} self.arg_params['cls_prob_label'] = mx.nd.zeros(arg_shapes_dict['cls_prob_label'], self.ctx) aux_names = self.symbol.list_auxiliary_states() self.aux_params = {k: mx.nd.zeros(s, self.ctx) for k, s in zip(aux_names, aux_shapes)} self.executor = self.symbol.bind(self.ctx, self.arg_params, args_grad=None, grad_req='null', aux_states=self.aux_params) output_dict = {name: nd for name, nd in zip(self.symbol.list_outputs(), self.executor.outputs)} self.executor.forward(is_train=False) scores = output_dict['cls_prob_output'].asnumpy() bbox_deltas = output_dict['bbox_pred_output'].asnumpy() pred_boxes = bbox_pred(roi_array[:, 1:], bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_array[0].shape[-2:]) if config.TEST.DEDUP_BOXES > 0: # map back to original scores = scores[inv_index, :] pred_boxes = pred_boxes[inv_index, :] return scores, pred_boxes
def main(): color = cv2.imread(args.img) # read image in b,g,r order img, scale = resize(color.copy(), 640, 1024) im_info = np.array([[img.shape[0], img.shape[1], scale]], dtype=np.float32) # (h, w, scale) img = np.swapaxes(img, 0, 2) img = np.swapaxes(img, 1, 2) # change to r,g,b order img = img[np.newaxis, :] # extend to (n, c, h, w) ctx = mx.gpu(args.gpu) _, arg_params, aux_params = mx.model.load_checkpoint(args.prefix, args.epoch) arg_params, aux_params = ch_dev(arg_params, aux_params, ctx) if 'resnet' in args.prefix: sym = resnet_50(num_class=2, bn_mom=0.99, bn_global=True, is_train=False) else: sym = get_vgg_test(num_classes=2) arg_params["data"] = mx.nd.array(img, ctx) arg_params["im_info"] = mx.nd.array(im_info, ctx) exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params) exe.forward(is_train=False) output_dict = {name: nd for name, nd in zip(sym.list_outputs(), exe.outputs)} rois = output_dict['rpn_rois_output'].asnumpy()[:, 1:] # first column is index scores = output_dict['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0] pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, (im_info[0][0], im_info[0][1])) cls_boxes = pred_boxes[:, 4:8] cls_scores = scores[:, 1] keep = np.where(cls_scores >= args.thresh)[0] cls_boxes = cls_boxes[keep, :] cls_scores = cls_scores[keep] dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32) keep = nms(dets.astype(np.float32), args.nms_thresh) dets = dets[keep, :] keep = nest(dets, thresh=args.nest_thresh) dets = dets[keep, :] for i in range(dets.shape[0]): bbox = dets[i, :4] cv2.rectangle(color, (int(round(bbox[0]/scale)), int(round(bbox[1]/scale))), (int(round(bbox[2]/scale)), int(round(bbox[3]/scale))), (0, 255, 0), 2) cv2.imwrite("result.jpg", color)
def im_detect(self, im_array, im_info=None, roi_array=None): """ perform detection of designated im, box, must follow minibatch.get_testbatch format :param im_array: numpy.ndarray [b c h w] :param im_info: numpy.ndarray [b 3] :param roi_array: numpy.ndarray [roi_num 5] :return: scores, pred_boxes """ # remove duplicate feature rois if config.TEST.DEDUP_BOXES > 0 and not config.TEST.HAS_RPN: roi_array = roi_array # rank roi by v .* (b, dx, dy, dw, dh) v = np.array([1, 1e3, 1e6, 1e9, 1e12]) # create hash and inverse index for rois hashes = np.round(roi_array * config.TEST.DEDUP_BOXES).dot(v) _, index, inv_index = np.unique(hashes, return_index=True, return_inverse=True) roi_array = roi_array[index, :] if self.executor is None: # fill in data if config.TEST.HAS_RPN: self.arg_params['data'] = mx.nd.array(im_array, self.ctx) self.arg_params['im_info'] = mx.nd.array(im_info, self.ctx) arg_shapes, out_shapes, aux_shapes = \ self.symbol.infer_shape(data=self.arg_params['data'].shape, im_info=self.arg_params['im_info'].shape) else: self.arg_params['data'] = mx.nd.array(im_array, self.ctx) self.arg_params['rois'] = mx.nd.array(roi_array, self.ctx) arg_shapes, out_shapes, aux_shapes = \ self.symbol.infer_shape(data=self.arg_params['data'].shape, rois=self.arg_params['rois'].shape) # fill in label and aux arg_shapes_dict = {name: shape for name, shape in zip(self.symbol.list_arguments(), arg_shapes)} self.arg_params['cls_prob_label'] = mx.nd.zeros(arg_shapes_dict['cls_prob_label'], self.ctx) aux_names = self.symbol.list_auxiliary_states() self.aux_params = {k: mx.nd.zeros(s, self.ctx) for k, s in zip(aux_names, aux_shapes)} # execute self.executor = self.symbol.bind(self.ctx, self.arg_params, args_grad=None, grad_req='null', aux_states=self.aux_params) executor = self.executor else: if config.TEST.HAS_RPN: # Test whether we need upsizing if np.prod(im_array.shape) > np.prod(self.executor.arg_dict['data'].shape): self.executor = self.executor.reshape(allow_up_sizing=True, data=im_array.shape) executor = self.executor else: executor = self.executor.reshape(allow_up_sizing=False, data=im_array.shape) # fill in data executor.arg_dict["data"][:] = im_array executor.arg_dict["im_info"][:] = im_info else: if np.prod(im_array.shape) > np.prod(self.executor.arg_dict['data'].shape) or\ np.prod(roi_array.shape) > np.prod(self.executor.arg_dict['rois'].shape): self.executor = self.executor.reshape(partial_shaping=True, allow_up_sizing=True, data=im_array.shape, rois=roi_array.shape) executor = self.executor else: executor = self.executor.reshape(partial_shaping=True, allow_up_sizing=False, data=im_array.shape, rois=roi_array.shape) # fill in data executor.arg_dict["data"][:] = im_array executor.arg_dict["rois"][:] = roi_array executor.forward(is_train=False) # save output scores = executor.output_dict['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = executor.output_dict['bbox_pred_reshape_output'].asnumpy()[0] if config.TEST.HAS_RPN: rois = executor.output_dict['rois_output'].asnumpy() rois = rois[:, 1:].copy() # scale back else: rois = roi_array[:, 1:] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_array[0].shape[-2:]) if config.TEST.DEDUP_BOXES > 0 and not config.TEST.HAS_RPN: # map back to original scores = scores[inv_index, :] pred_boxes = pred_boxes[inv_index, :] return scores, pred_boxes
def forward(self, is_train, req, in_data, out_data, aux): # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) pre_nms_topN = config[self.cfg_key].RPN_PRE_NMS_TOP_N post_nms_topN = config[self.cfg_key].RPN_POST_NMS_TOP_N nms_thresh = config[self.cfg_key].RPN_NMS_THRESH min_size = config[self.cfg_key].RPN_MIN_SIZE # the first set of anchors are background probabilities # keep the second part scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :] bbox_deltas = in_data[1].asnumpy() im_info = in_data[2].asnumpy()[0, :] if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) # 1. Generate proposals from bbox_deltas and shifted anchors height, width = scores.shape[-2:] if DEBUG: print 'score map size: {}'.format(scores.shape) # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape( (1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # Convert anchors into proposals via bbox transformations proposals = bbox_pred(anchors, bbox_deltas) # 2. clip predicted boxes to image proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = ProposalOperator._filter_boxes(proposals, min_size * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) keep = nms(np.hstack((proposals, scores)), nms_thresh) if post_nms_topN > 0: keep = keep[:post_nms_topN] # pad to ensure output size remains unchanged if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] # Output rois array # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def forward(self, is_train, req, in_data, out_data, aux): # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) pre_nms_topN = config[self.cfg_key].RPN_PRE_NMS_TOP_N post_nms_topN = config[self.cfg_key].RPN_POST_NMS_TOP_N nms_thresh = config[self.cfg_key].RPN_NMS_THRESH min_size = config[self.cfg_key].RPN_MIN_SIZE # the first set of anchors are background probabilities # keep the second part scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :] if np.isnan(scores).any(): raise ValueError("there is nan in input scores") bbox_deltas = in_data[1].asnumpy() if np.isnan(bbox_deltas).any(): raise ValueError("there is nan in input bbox_deltas") im_info = in_data[2].asnumpy()[0, :] if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) # 1. Generate proposals from bbox_deltas and shifted anchors height, width = scores.shape[-2:] if self.cfg_key == 'TRAIN': height, width = int(im_info[0] / self._feat_stride), int(im_info[1] / self._feat_stride) if DEBUG: print 'score map size: {}'.format(scores.shape) print "resudial = ", scores.shape[2] - height, scores.shape[3] - width # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # Convert anchors into proposals via bbox transformations proposals = bbox_pred(anchors, bbox_deltas) # 2. clip predicted boxes to image proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = ProposalOperator._filter_boxes(proposals, min_size * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) keep = nms(np.hstack((proposals, scores)), nms_thresh) if post_nms_topN > 0: keep = keep[:post_nms_topN] # pad to ensure output size remains unchanged if len(keep) < post_nms_topN: if len(keep) == 0: logging.log(logging.ERROR, "currently len(keep) is zero") pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] # Output rois array # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))