def get_image(roidb, config): """ preprocess image and return processed roidb :param roidb: a list of roidb :return: list of img as in mxnet format roidb add new item['im_info'] 0 --- x (width, second dim of im) | y (height, first dim of im) """ num_images = len(roidb) processed_ims = [] processed_roidb = [] for i in range(num_images): roi_rec = roidb[i] assert os.path.exists(roi_rec['image']), '%s does not exist'.format(roi_rec['image']) im = cv2.imread(roi_rec['image'], cv2.IMREAD_COLOR|cv2.IMREAD_IGNORE_ORIENTATION) if roidb[i]['flipped']: im = im[:, ::-1, :] new_rec = roi_rec.copy() scale_ind = random.randrange(len(config.SCALES)) target_size = config.SCALES[scale_ind][0] max_size = config.SCALES[scale_ind][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) processed_ims.append(im_tensor) im_info = [im_tensor.shape[2], im_tensor.shape[3], im_scale] new_rec['boxes'] = clip_boxes(np.round(roi_rec['boxes'].copy() * im_scale), im_info[:2]) new_rec['im_info'] = im_info processed_roidb.append(new_rec) return processed_ims, processed_roidb
def im_detect(predictor, data_batch, data_names, scale): st = time.time() output = predictor.predict(data_batch) et = time.time() print 'predict{:.4f}s'.format(et - st) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy().reshape((-1, 5))[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale et = time.time() print 'im_detect{:.4f}s'.format(et - st) return scores, pred_boxes, data_dict
def im_detect(predictor, data_batch, data_names, scale): output = predictor.predict(data_batch) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy().reshape((-1, 5))[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] import ipdb ipdb.set_trace() # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale if config.HAS_PART: head_scores = output['head_prob_reshape_output'].asnumpy()[0] head_gids = np.argmax(head_scores, axis=1) head_deltas = output['head_pred_reshape_output'].asnumpy()[0] # means = config.TRAIN.BBOX_MEANS stds = np.reshape(np.array(config.TRAIN.BBOX_STDS), (-1, 4)) head_deltas *= np.tile(stds, (1, head_scores.shape[1])) head_boxes = pred_head(rois, head_deltas, head_gids, config.PART_GRID_HW) head_boxes /= scale joints_scores = [ output['joint_prob{}_reshape_output'.format(i)].asnumpy()[0] for i in range(4) ] joints_gids = [np.argmax(j, axis=1) for j in joints_scores] joints_deltas = [ output['joint_pred{}_reshape_output'.format(i)].asnumpy()[0] for i in range(4) ] joints_deltas = [ j * np.tile(stds[:, :2], (1, head_scores.shape[1])) for j in joints_deltas ] joints = [pred_joint(rois, jd, jid, config.PART_GRID_HW) \ for (jd, jid) in zip(joints_deltas, joints_gids)] joints = np.hstack(joints) joints /= scale return scores, pred_boxes, head_boxes, joints, data_dict return scores, pred_boxes, data_dict
def im_detect(self, im_array, im_info=None, roi_array=None): """ perform detection of designated im, box, must follow minibatch.get_testbatch format :param im_array: numpy.ndarray [b c h w] :param im_info: numpy.ndarray [b 3] :param roi_array: numpy.ndarray [roi_num 5] :return: scores, pred_boxes """ # fill in data if config.TEST.HAS_RPN: self.arg_params['data'] = mx.nd.array(im_array, self.ctx) self.arg_params['im_info'] = mx.nd.array(im_info, self.ctx) arg_shapes, out_shapes, aux_shapes = \ self.symbol.infer_shape(data=self.arg_params['data'].shape, im_info=self.arg_params['im_info'].shape) else: self.arg_params['data'] = mx.nd.array(im_array, self.ctx) self.arg_params['rois'] = mx.nd.array(roi_array, self.ctx) arg_shapes, out_shapes, aux_shapes = \ self.symbol.infer_shape(data=self.arg_params['data'].shape, rois=self.arg_params['rois'].shape) # fill in label arg_shapes_dict = { name: shape for name, shape in zip(self.symbol.list_arguments(), arg_shapes) } self.arg_params['cls_prob_label'] = mx.nd.zeros( arg_shapes_dict['cls_prob_label'], self.ctx) # execute self.executor = self.symbol.bind(self.ctx, self.arg_params, args_grad=None, grad_req='null', aux_states=self.aux_params) output_dict = { name: nd for name, nd in zip(self.symbol.list_outputs(), self.executor.outputs) } self.executor.forward(is_train=False) # save output scores = output_dict['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output_dict['bbox_pred_reshape_output'].asnumpy()[0] if config.TEST.HAS_RPN: rois = output_dict['rois_output'].asnumpy()[:, 1:] else: rois = roi_array[:, 1:] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_array[0].shape[-2:]) return scores, pred_boxes
def coco_results_one_category_kernel(data_pack): cat_id = data_pack['cat_id'] ann_type = data_pack['ann_type'] binary_thresh = data_pack['binary_thresh'] all_im_info = data_pack['all_im_info'] boxes = data_pack['boxes'] if ann_type == 'bbox': masks = [] elif ann_type == 'segm': masks = data_pack['masks'] else: print 'unimplemented ann_type: ' + ann_type cat_results = [] for im_ind, im_info in enumerate(all_im_info): index = im_info['index'] try: dets = boxes[im_ind].astype(np.float) except: dets = boxes[im_ind] if len(dets) == 0: continue scores = dets[:, -1] if ann_type == 'bbox': xs = dets[:, 0] ys = dets[:, 1] ws = dets[:, 2] - xs + 1 hs = dets[:, 3] - ys + 1 result = [{ 'image_id': index, 'category_id': cat_id, 'bbox': [xs[k], ys[k], ws[k], hs[k]], 'score': scores[k] } for k in xrange(dets.shape[0])] elif ann_type == 'segm': width = im_info['width'] height = im_info['height'] dets[:, :4] = clip_boxes(dets[:, :4], [height, width]) mask_encode = mask_voc2coco(masks[im_ind], dets[:, :4], height, width, binary_thresh) result = [{ 'image_id': index, 'category_id': cat_id, 'segmentation': mask_encode[k], 'score': scores[k] } for k in xrange(len(mask_encode))] cat_results.extend(result) return cat_results
def im_detect(predictor, data_batch, data_names): output = predictor.predict(data_batch) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy()[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) return scores, pred_boxes, data_dict
def im_detect(predictor, data_batch, data_names, scale): output = predictor.predict(data_batch) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy().reshape((-1, 5))[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale return scores, pred_boxes, data_dict
def im_rpn_detect(predictor, data_batch, data_names, scale): output = predictor.predict(data_batch) print(output.keys()) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: rois = data_dict['rois'].asnumpy().reshape((-1, 5))[:, 1:] im_shape = data_dict['data'].shape # save output scores = output['rois_score'].asnumpy() #bbox_deltas = output['rpn_bbox_pred_output'].asnumpy()[0] # # post processing #pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(rois, im_shape[-2:]) #print(pred_boxes.shape) # # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale return scores, pred_boxes, data_dict
def im_detect(predictor, data_batch, data_names, scales): output_all = predictor.predict(data_batch) data_dict_all = [ dict(zip(data_names, data_batch.data[i])) for i in xrange(len(data_batch.data)) ] scores_all = [] pred_boxes_all = [] pred_masks_all = [] rois_all = [] for output, data_dict, scale in zip(output_all, data_dict_all, scales): if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: raise NotImplementedError im_shape = data_dict['data'].shape # save output scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] pred_masks = output['mask_pred_output'].asnumpy() # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back rois = rois / scale pred_boxes = pred_boxes / scale #print scores.shape, rois.shape, pred_boxes.shape, pred_masks.shape scores_all.append(scores) rois_all.append(rois) pred_boxes_all.append(pred_boxes) pred_masks_all.append(pred_masks) return scores_all, rois_all, pred_boxes_all, pred_masks_all, data_dict_all
def im_detect_mask(predictor, data_batch, data_names, scale=1): output = predictor.predict(data_batch) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: raise NotImplementedError im_shape = data_dict['data'].shape if config.TEST.HAS_RPN: scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() else: raise NotImplementedError # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale return scores, pred_boxes, data_dict, mask_output
def im_detect_mask(predictor, data_batch, data_names, scale=1): output = predictor.predict(data_batch) data_dict = dict(zip(data_names, data_batch.data)) if config.TEST.HAS_RPN: rois = output['rois_output'].asnumpy()[:, 1:] else: raise NotImplementedError im_shape = data_dict['data'].shape if config.TEST.HAS_RPN: scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() else: raise NotImplementedError # post processing pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, im_shape[-2:]) # we used scaled image & roi to train, so it is necessary to transform them back pred_boxes = pred_boxes / scale return scores, pred_boxes, data_dict, mask_output
def demo_net(predictor, data, image_names, im_scales): data = [[mx.nd.array(data[i][name]) for name in DATA_NAMES] for i in xrange(len(data))] # warm up for i in xrange(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(DATA_NAMES, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] _, _, _, _, _= im_detect(predictor, data_batch, DATA_NAMES, scales) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(DATA_NAMES, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in xrange(len(data_batch.data))] tic() scores, boxes, boxes2, masks, data_dict = im_detect(predictor, data_batch, DATA_NAMES, scales) im_shapes = [data_batch.data[i][0].shape[2:4] for i in xrange(len(data_batch.data))] # mask output if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in xrange(config.NUM_CLASSES)] all_masks = [[] for _ in xrange(config.NUM_CLASSES)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, config.NUM_CLASSES): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, config.NUM_CLASSES)] masks = [all_masks[j] for j in range(1, config.NUM_CLASSES)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print (im_height, im_width) boxes_ = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting(masks, boxes_, scores[0], config.NUM_CLASSES, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, 0) dets = [result_dets[j] for j in range(1, config.NUM_CLASSES)] masks = [result_masks[j][:, 0, :, :] for j in range(1, config.NUM_CLASSES)] print 'testing {} {:.4f}s'.format(im_name, toc()) # visualize for i in xrange(len(dets)): keep = np.where(dets[i][:,-1]>0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread('../data/demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, CLASSES) # debug ''' for ii in range(scores[0].shape[0]): for jj in range(1, scores[0].shape[1]): if scores[0][ii][jj]>0.7: print ii, jj, scores[0][ii][jj] ''' # bounding box output all_boxes = [[] for _ in CLASSES] nms = py_nms_wrapper(NMS_THRESH) for cls in CLASSES: cls_ind = CLASSES.index(cls)+1 cls_boxes = boxes2[0][:, 4 * cls_ind:4 * (cls_ind + 1)] cls_scores = scores[0][:, cls_ind, np.newaxis] keep = np.where(cls_scores >= CONF_THRESH)[0] #print cls, keep dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :] keep = nms(dets) all_boxes[cls_ind-1] = dets[keep, :] boxes_this_image = [all_boxes[j] for j in range(len(CLASSES))] vis_all_detection(data_dict[0]['data'].asnumpy(), boxes_this_image, CLASSES, im_scales[idx]) print 'done'
def forward(self, is_train, req, in_data, out_data, aux): """Implements forward computation. is_train : bool, whether forwarding for training or testing. req : list of {'null', 'write', 'inplace', 'add'}, how to assign to out_data. 'null' means skip assignment, etc. in_data : list of NDArray, input data. out_data : list of NDArray, pre-allocated output buffers. aux : list of NDArray, mutable auxiliary states. Usually not used. """ nms = gpu_nms_wrapper(self._threshold, in_data[0].context.device_id) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError( "Sorry, multiple images each device is not implemented") # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) # 对(H,W)大小的特征图上的每一点i: # 以 i 为中心生成A个锚定框 # 利用回归的位置参数,修正这 A 个 anchor 的位置,得到 RoIs # 将预测的边界框裁剪成图像 # 清除掉预测边界框中长或宽 小于阈值的 # 按分数降序排列(proposal,score) # 在采用NMS取前N个预测边界框 # 使用阈值0.7对这N个框使用非极大值抑制 # 取使用NMS后前n个预测边界框 # 返回前Top n 个的边界框,进行分类和回归 pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size = self._rpn_min_size # the first set of anchors are background probabilities # keep the second part scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :] bbox_deltas = in_data[1].asnumpy() im_info = in_data[2].asnumpy()[0, :] logger.debug('im_info: %s' % im_info) # 1. Generate proposals from bbox_deltas and shifted anchors # use real image size instead of padded feature map sizes height, width = int(im_info[0] / self._feat_stride), int( im_info[1] / self._feat_stride) logger.debug('score map size: (%d, %d)' % (scores.shape[2], scores.shape[3])) logger.debug('resudial: (%d, %d)' % (scores.shape[2] - height, scores.shape[3] - width)) # Enumerate all shifts # 这块的思路是生成一系列的shift, 然后每一个shift和9个anchor相加,迭代出每一个位置的9个框 shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) #产生一个以向量x为行,向量y为列的矩阵 #经过meshgrid shift_x = [[ 0 16 32 ..., 560 576 592] [ 0 16 32 ..., 560 576 592] [ 0 16 32 ..., 560 576 592] ..., [ 0 16 32 ..., 560 576 592] [ 0 16 32 ..., 560 576 592] [ 0 16 32 ..., 560 576 592]] #shift_y = [[ 0 0 0 ..., 0 0 0] [ 16 16 16 ..., 16 16 16] [ 32 32 32 ..., 32 32 32] ..., [560 560 560 ..., 560 560 560] [576 576 576 ..., 576 576 576] [592 592 592 ..., 592 592 592]] shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # 转至之后形成所有位移 # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] # _anchors中每一个anchor和每一个shift相加得出结果 anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape( (1, K, 4)).transpose((1, 0, 2)) # K个位移,每个位移A个框 anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # Convert anchors into proposals via bbox transformations # 根据回归的偏移量修正位置 proposals = bbox_pred(anchors, bbox_deltas) # 2. clip predicted boxes to image # 裁剪掉边框超出图片边界的部分 proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) # 清除掉预测边界框中长或宽 小于阈值的 keep = self._filter_boxes(proposals, min_size * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) # 按分数降序排列,并取前N个(proposal, score) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) det = np.hstack((proposals, scores)).astype(np.float32) keep = nms(det) if post_nms_topN > 0: keep = keep[:post_nms_topN] # pad to ensure output size remains unchanged # 如果不够,就随机选择不足的个数来填充 if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] # Output rois array # 输出ROIS,送给fast-rcnn训练 # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) # 形成五元组(0,x1,y1,x2,y2) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0].context.device_id) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError("Sorry, multiple images each device is not implemented") # for each (H, W) location i # generate A anchor boxes centered on cell i # apply predicted bbox deltas at cell i to each of the A anchors # clip predicted boxes to image # remove predicted boxes with either height or width < threshold # sort all (proposal, score) pairs by score from highest to lowest # take top pre_nms_topN proposals before NMS # apply NMS with threshold 0.7 to remaining proposals # take after_nms_topN proposals after NMS # return the top proposals (-> RoIs top, scores top) pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size = self._rpn_min_size # the first set of anchors are background probabilities # keep the second part scores = in_data[0].asnumpy()[:, self._num_anchors:, :, :] bbox_deltas = in_data[1].asnumpy() im_info = in_data[2].asnumpy()[0, :] if DEBUG: print('im_size: ({}, {})'.format(im_info[0], im_info[1])) print('scale: {}'.format(im_info[2])) # 1. Generate proposals from bbox_deltas and shifted anchors # use real image size instead of padded feature map sizes height, width = int(im_info[0] / self._feat_stride), int(im_info[1] / self._feat_stride) if DEBUG: print('score map size: {}'.format(scores.shape)) print("resudial: {}".format((scores.shape[2] - height, scores.shape[3] - width))) # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride shift_y = np.arange(0, height) * self._feat_stride shift_x, shift_y = np.meshgrid(shift_x, shift_y) shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(), shift_y.ravel())).transpose() # Enumerate all shifted anchors: # # add A anchors (1, A, 4) to # cell K shifts (K, 1, 4) to get # shift anchors (K, A, 4) # reshape to (K*A, 4) shifted anchors A = self._num_anchors K = shifts.shape[0] anchors = self._anchors.reshape((1, A, 4)) + shifts.reshape((1, K, 4)).transpose((1, 0, 2)) anchors = anchors.reshape((K * A, 4)) # Transpose and reshape predicted bbox transformations to get them # into the same order as the anchors: # # bbox deltas will be (1, 4 * A, H, W) format # transpose to (1, H, W, 4 * A) # reshape to (1 * H * W * A, 4) where rows are ordered by (h, w, a) # in slowest to fastest order bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) # Same story for the scores: # # scores are (1, A, H, W) format # transpose to (1, H, W, A) # reshape to (1 * H * W * A, 1) where rows are ordered by (h, w, a) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # Convert anchors into proposals via bbox transformations proposals = bbox_pred(anchors, bbox_deltas) # 2. clip predicted boxes to image proposals = clip_boxes(proposals, im_info[:2]) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_info[2]) keep = self._filter_boxes(proposals, min_size * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] # 4. sort all (proposal, score) pairs by score from highest to lowest # 5. take top pre_nms_topN (e.g. 6000) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] # 6. apply nms (e.g. threshold = 0.7) # 7. take after_nms_topN (e.g. 300) # 8. return the top proposals (-> RoIs top) det = np.hstack((proposals, scores)).astype(np.float32) keep = nms(det) if post_nms_topN > 0: keep = keep[:post_nms_topN] # pad to ensure output size remains unchanged if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] # Output rois array # Our RPN implementation only supports a single input image, so all # batch inds are 0 batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): #print('in_detect', threshold, scales, do_flip, do_nms) proposals_list = [] scores_list = [] landmarks_list = [] strides_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] imgs = [img] if isinstance(img, list): imgs = img for img in imgs: for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() if flip: im = im[:, ::-1, :] if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 3), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X1 uses', diff.total_seconds(), 'seconds') #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) #im_info = [im.shape[0], im.shape[1], im_scale] im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = ( im[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X2 uses', diff.total_seconds(), 'seconds') data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X3 uses', diff.total_seconds(), 'seconds') self.model.forward(db, is_train=False) net_out = self.model.get_outputs() #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn sym_idx = 0 for _idx, s in enumerate(self._feat_stride_fpn): #if len(scales)>1 and s==32 and im_scale==scales[-1]: # continue _key = 'stride%s' % s stride = int(s) is_cascade = False if self.cascade: is_cascade = True #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): # continue #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[sym_idx].asnumpy() if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('A uses', diff.total_seconds(), 'seconds') #print(scores.shape) #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[sym_idx + 1].asnumpy() #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[ 2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) #print('HW', (height, width), file=sys.stderr) #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) #print('anchors', anchors.shape, file=sys.stderr) #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) #print('scores', scores.shape, file=sys.stderr) #scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape( (-1, 1)) #print('pre', bbox_deltas.shape, height, width) #bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A #print(bbox_deltas.shape) bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0:: 4] * self.bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1:: 4] * self.bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2:: 4] * self.bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3:: 4] * self.bbox_stds[3] proposals = self.bbox_pred(anchors, bbox_deltas) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) if is_cascade: cascade_sym_num = 0 cls_cascade = False bbox_cascade = False __idx = [3, 4] if not self.use_landmarks: __idx = [2, 3] for diff_idx in __idx: if sym_idx + diff_idx >= len(net_out): break body = net_out[sym_idx + diff_idx].asnumpy() if body.shape[1] // A == 2: #cls branch if cls_cascade or bbox_cascade: break else: cascade_scores = body[:, self. _num_anchors[ 'stride%s' % s]:, :, :] cascade_scores = cascade_scores.transpose( (0, 2, 3, 1)).reshape((-1, 1)) #scores = (scores+cascade_scores)/2.0 scores = cascade_scores #TODO? cascade_sym_num += 1 cls_cascade = True #print('find cascade cls at stride', stride) elif body.shape[1] // A == 4: #bbox branch cascade_deltas = body.transpose( (0, 2, 3, 1)).reshape( (-1, bbox_pred_len)) cascade_deltas[:, 0:: 4] = cascade_deltas[:, 0:: 4] * self.bbox_stds[ 0] cascade_deltas[:, 1:: 4] = cascade_deltas[:, 1:: 4] * self.bbox_stds[ 1] cascade_deltas[:, 2:: 4] = cascade_deltas[:, 2:: 4] * self.bbox_stds[ 2] cascade_deltas[:, 3:: 4] = cascade_deltas[:, 3:: 4] * self.bbox_stds[ 3] proposals = self.bbox_pred( proposals, cascade_deltas) cascade_sym_num += 1 bbox_cascade = True #print('find cascade bbox at stride', stride) proposals = clip_boxes(proposals, im_info[:2]) #if self.vote: # if im_scale>1.0: # keep = self._filter_boxes2(proposals, 160*im_scale, -1) # else: # keep = self._filter_boxes2(proposals, -1, 100*im_scale) # if stride==4: # keep = self._filter_boxes2(proposals, 12*im_scale, -1) # proposals = proposals[keep, :] # scores = scores[keep] #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 scores_ravel = scores.ravel() #print('__shapes', proposals.shape, scores_ravel.shape) #print('max score', np.max(scores_ravel)) order = np.where(scores_ravel >= threshold)[0] #_scores = scores_ravel[order] #_order = _scores.argsort()[::-1] #order = order[_order] proposals = proposals[order, :] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if self.nms_threshold < 0.0: _strides = np.empty(shape=(scores.shape), dtype=np.float32) _strides.fill(stride) strides_list.append(_strides) if not self.vote and self.use_landmarks: landmark_deltas = net_out[sym_idx + 2].asnumpy() #landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmark_deltas *= self.landmark_std #print(landmark_deltas.shape, landmark_deltas) landmarks = self.landmark_pred( anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 #for a in range(5): # oldx1 = landmarks[:, a].copy() # landmarks[:,a] = im.shape[1] - oldx1 - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] #flandmarks[:, idx*2] = landmarks[:,a*2] #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale #landmarks /= im_scale #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) landmarks_list.append(landmarks) #proposals = np.hstack((proposals, landmarks)) if self.use_landmarks: sym_idx += 3 else: sym_idx += 2 if is_cascade: sym_idx += cascade_sym_num if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('B uses', diff.total_seconds(), 'seconds') proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) if self.nms_threshold < 0.0: return np.zeros((0, 6)), landmarks else: return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) #print('shapes', proposals.shape, scores.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] proposals = proposals[order, :] scores = scores[order] if self.nms_threshold < 0.0: strides = np.vstack(strides_list) strides = strides[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) if self.nms_threshold > 0.0: pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) elif self.nms_threshold < 0.0: det = np.hstack( (proposals[:, 0:4], scores, strides)).astype(np.float32, copy=False) else: det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('C uses', diff.total_seconds(), 'seconds') return det, landmarks
def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0][0].context.device_id) cls_prob_dict = dict(zip(self.fpn_keys, in_data[0:len(self.fpn_keys)])) bbox_pred_dict = dict( zip(self.fpn_keys, in_data[len(self.fpn_keys):2 * len(self.fpn_keys)])) #for i in xrange(6): # print(i, in_data[i].asnumpy().shape) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError( "Sorry, multiple images each device is not implemented") pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size_dict = self._rpn_min_size_fpn proposals_list = [] scores_list = [] for s in self._feat_stride_fpn: _key = 'stride%s' % s stride = int(s) scores = cls_prob_dict[_key].asnumpy() #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = bbox_pred_dict['stride%s' % s].asnumpy() im_info = in_data[-1].asnumpy()[0, :] #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane( height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) #proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] #if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) #if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self._threshold < 1.0: keep = nms(det) else: keep = range(det.shape[0]) #print(det.shape, len(keep), post_nms_topN) if post_nms_topN > 0: keep = keep[:post_nms_topN] #print(det.shape, len(keep), post_nms_topN) num_keep = len(keep) #print('keep', keep, file=sys.stderr) if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] scores[num_keep:, :] = -1.0 #print('333 proposals', proposals[0:5,:], file=sys.stderr) #print('det', det.shape, num_keep) #print('first proposal', proposals[0], file=sys.stderr) batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def detect(self, img, threshold=0.5, im_scale=1.0): proposals_list = [] scores_list = [] landmarks_list = [] data = nd.array(img) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s stride = int(s) if self.use_landmarks: idx = _idx * 3 else: idx = _idx * 2 scores = net_out[idx].asnumpy() scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, (img.shape[2], img.shape[3])) scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape((-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) return det, landmarks
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): proposals_list = [] scores_list = [] landmarks_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() if flip: im = im[:, ::-1, :] if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 3), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X1 uses', diff.total_seconds(), 'seconds') im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = ( im[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X2 uses', diff.total_seconds(), 'seconds') data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X3 uses', diff.total_seconds(), 'seconds') self.model.forward(db, is_train=False) net_out = self.model.get_outputs() for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s stride = int(s) if self.use_landmarks: idx = _idx * 3 else: idx = _idx * 2 scores = net_out[idx].asnumpy() if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('A uses', diff.total_seconds(), 'seconds') scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = self._clip_pad( landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('B uses', diff.total_seconds(), 'seconds') proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) #print('shapes', proposals.shape, scores.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] proposals = proposals[order, :] scores = scores[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) #if self.use_landmarks: # det = np.hstack((det, landmarks)) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('C uses', diff.total_seconds(), 'seconds') return det, landmarks
def detect(self, img, threshold=0.5): proposals_list = [] scores_list = [] landmarks_list = [] im_info = [640, 640] if img.shape[0] != img.shape[1]: BLUE = (255, 0, 0) if img.shape[0] > img.shape[1]: img = cv2.copyMakeBorder(img, 0, 0, 0, img.shape[0] - img.shape[1], cv2.BORDER_CONSTANT, value=BLUE) else: img = cv2.copyMakeBorder(img, 0, img.shape[1] - img.shape[0], 0, 0, cv2.BORDER_CONSTANT, value=BLUE) re_scale = float(im_info[0]) / float(img.shape[0]) img = cv2.resize(img, (im_info[0], im_info[1])) img = img.astype(np.float32) im_tensor = np.zeros((1, 3, img.shape[0], img.shape[1])) for i in range(3): im_tensor[ 0, i, :, :] = (img[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() for _idx, s in enumerate(self._feat_stride_fpn): stride = int(s) idx = _idx * 3 scores = net_out[idx].asnumpy() scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() idx += 1 landmark_deltas = net_out[idx].asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) landmark_deltas = self._clip_pad(landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] scores = scores[order] proposals = proposals[order, :] landmarks = landmarks[order, :] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 proposals[:, 0:4] /= re_scale landmarks[:, :, 0:2] /= re_scale scores_list.append(scores) proposals_list.append(proposals) landmarks_list.append(landmarks) scores = np.vstack(scores_list) proposals = np.vstack(proposals_list) landmarks = np.vstack(landmarks_list) if proposals.shape[0] == 0: return np.zeros((0, 5)), np.zeros((0, 5, 2)) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] scores = scores[order] proposals = proposals[order, :] landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] landmarks = landmarks[keep] return det, landmarks
def detect(self, img, thresh, scales=[1.0], do_flip=False): proposal_list = [] scores_list = [] landmarks_list = [] flips = [0,1] if do_flip else [0] for im_scale in scales: for flip in flips: if im_scale!=1.0: img = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: img = img.copy() if flip: img = img[:,::-1,:] # img = self.pad_img_to_32(img) imgshape = [img.shape[0], img.shape[1]] img = self.preprocess(img) img = torch.from_numpy(img) if self.use_gpu: img = img.cuda() net_out = self.model(img) for _idx,s in enumerate(self.fpn_keys): idx = _idx * 3 scores = net_out[idx].detach().cpu().numpy() scores = scores[:, self._num_anchors[s]:] idx += 1 bbox_deltas = net_out[idx].detach().cpu().numpy() h, w = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors[s] K = h*w anchors_fpn = self._anchors_fpn[s] anchors_fpn = np.float32(anchors_fpn) anchors = anchors_plane(h, w, s, anchors_fpn) anchors = anchors.reshape((K*A, 4)) scores = self._clip_pad(scores, (h, w)) scores = scores.transpose([0,2,3,1]).reshape([-1,1]) # print('SCR') # print(scores) # print(scores.shape) # input() bbox_deltas = self._clip_pad(bbox_deltas, (h,w)) bbox_deltas = bbox_deltas.transpose([0,2,3,1]) bbox_pred_len = bbox_deltas.shape[3]//A bbox_deltas = bbox_deltas.reshape([-1, bbox_pred_len]) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, imgshape) scores_ravel = scores.ravel() order = np.where(scores_ravel>=thresh)[0] proposals = proposals[order] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:,:4] /= im_scale # print('proposals') # print(proposals) # print(proposals.shape) # input() proposal_list.append(proposals) scores_list.append(scores) # landmarks idx += 1 landmark_deltas = net_out[idx].detach().cpu().numpy() landmark_deltas = self._clip_pad(landmark_deltas, (h,w)) landmark_pred_len = landmark_deltas.shape[1]//A landmark_deltas = landmark_deltas.transpose((0,2,3,1)).reshape([-1,5,landmark_pred_len//5]) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:,:,0] = imgshape[1] - landmarks[:,:,0] - 1 order = [1,0,2,4,3] flandmarks = landmarks[:,np.int32(order)] landmarks[:,:,:2] /= im_scale landmarks_list.append(landmarks) # print('PROPOSAL', proposal_list) proposals = np.vstack(proposal_list) landmarks = None if proposals.shape[0]==0: return np.zeros([0,5]), np.zeros([0,5,2]) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order] scores = scores[order] landmarks = np.vstack(landmarks_list) landmarks = np.float32(landmarks[order]) pre_det = np.hstack([proposals[:, 0:4], scores]) pre_det = np.float32(pre_det) keep = self.nms(pre_det) det = np.hstack([pre_det, proposals[:,4:]]) det = det[keep] landmarks = landmarks[keep] return det, landmarks
def detect(self, img, threshold=0.05, scales=[1.0]): proposals_list = [] scores_list = [] for im_scale in scales: if im_scale!=1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn for s in self._feat_stride_fpn: if len(scales)>1 and s==32 and im_scale==scales[-1]: continue _key = 'stride%s'%s stride = int(s) idx = 0 if s==16: idx=2 elif s==8: idx=4 print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() #print(scores.shape) idx+=1 #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s'%s]:, :, :] bbox_deltas = net_out[idx].asnumpy() #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s'%s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32)) #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) #proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals /= im_scale proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] #if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) #if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self.nms_threshold<1.0: keep = self.nms(det) det = det[keep, :] if threshold>0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] return det
def detect(self, img, threshold=0.5, scales=[1.0]): proposals_list = [] proposals_kp_list = [] scores_list = [] for im_scale in scales: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) # im_shape = im.shape # self.model.bind(data_shapes=[('data', (1, 3, im_shape[0], im_shape[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] #bgr2rgb mxnet rgb opencv bgr data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) timea = datetime.datetime.now() self.model.forward(db, is_train=False) timeb = datetime.datetime.now() diff = timeb - timea print('forward uses', diff.total_seconds(), 'seconds') net_out = self.model.get_outputs() #网络的输出为len=9的list,针对三个不同的stride,分为三大块的list,其中每个list分别代表score,bbox,kpoint三个维度的结果, pre_nms_topN = self._rpn_pre_nms_top_n #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn for s in self.feat_strides: _key = 'stride%s' % s # print(_key) stride = int(s) if s == self.feat_strides[0]: idx = 0 if s == self.feat_strides[1]: idx = 3 elif s == self.feat_strides[2]: idx = 6 # print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() #获取每个stride下的分类得分 idx += 1 # print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s'%s]:, :, :] #去掉了其中lable的值??? bbox_deltas = net_out[idx].asnumpy() idx += 1 _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] # kpoint kpoint_deltas = net_out[idx].asnumpy() A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) #RP映射回原图中的坐标位置 # print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) # print('predict bbox_deltas', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) # print('after clip pad', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) kpoint_deltas = self._clip_pad(kpoint_deltas, (height, width)) kpoint_deltas = kpoint_deltas.transpose((0, 2, 3, 1)).reshape((-1, 10)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) #将超出图像的坐标去除掉 proposals_kp = kpoint_pred(anchors, kpoint_deltas) proposals_kp = clip_points(proposals_kp, im_info[:2]) #取出score的top N scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] proposals_kp = proposals_kp[order, :] scores = scores[order] proposals /= im_scale proposals_kp /= im_scale proposals_list.append(proposals) proposals_kp_list.append(proposals_kp) scores_list.append(scores) proposals = np.vstack(proposals_list) proposals_kp = np.vstack(proposals_kp_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] #if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] proposals_kp = proposals_kp[order, :] scores = scores[order] det = np.hstack((proposals, scores, proposals_kp)).astype(np.float32) #if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self.nms_threshold < 1.0: keep = self.nms(det) det = det[keep, :] if threshold > 0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] return det
def detect(self, img, scales_index=0): proposals_list = [] scores_list = [] im_src = img.copy() CONSTANT = config.TEST.CONSTANT BLACK = [0, 0, 0] img = cv2.copyMakeBorder(img, CONSTANT, CONSTANT, CONSTANT, CONSTANT, cv2.BORDER_CONSTANT, value=BLACK) scales = self.get_boxes(img, scales_index) for im_scale in scales: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) # self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data,), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n for s in self._feat_stride_fpn: if len(scales) > 1 and s == 32 and im_scale == scales[-1]: continue _key = 'stride%s' % s stride = int(s) idx = 0 if s == 16: idx = 2 elif s == 8: idx = 4 # print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() # print(scores.shape) idx += 1 # print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[idx].asnumpy() _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) # print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) # print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) # print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) # proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals /= im_scale # #add by sai with pyramidbox to filt scale face # if im_scale > 1: # index = np.where( # np.minimum(proposals[:, 2] - proposals[:, 0] + 1, # proposals[:, 3] - proposals[:, 1] + 1) < 50)[0] # proposals = proposals[index, :] # scores = scores[index, :] # else: # index = np.where( # np.maximum(proposals[:, 2] - proposals[:, 0] + 1, # proposals[:, 3] - proposals[:, 1] + 1) > 20)[0] # proposals = proposals[index, :] # scores = scores[index, :] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) if self.nms_threshold < 1.0: keep = self.nms(det) det = det[keep, :] threshold = config.TEST.SCORE_THRESH if threshold > 0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] # add by sai if det.shape[0] != 0: for i in range(det.shape[0]): det[i, :][0] = det[i, :][0] - CONSTANT det[i, :][1] = det[i, :][1] - CONSTANT det[i, :][2] = det[i, :][2] - CONSTANT det[i, :][3] = det[i, :][3] - CONSTANT if det[i, :][0] < 0: det[i, :][0] = 0 if det[i, :][2] > im_src.shape[1]: det[i, :][2] = im_src.shape[1] if det[i, :][1] < 0: det[i, :][1] = 0 if det[i, :][3] > im_src.shape[0]: det[i, :][3] = im_src.shape[0] return det
def demo_maskrcnn(network, ctx, prefix, epoch, vis=True, has_rpn=True, thresh=0.001): assert has_rpn, "Only has_rpn==True has been supported." sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS) arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True) split = False max_image_shape = (1, 3, 1024, 1024) max_data_shapes = [("data", max_image_shape), ("im_info", (1, 3))] mod = MutableModule(symbol=sym, data_names=["data", "im_info"], label_names=None, max_data_shapes=max_data_shapes, context=ctx) mod.bind(data_shapes=max_data_shapes, label_shapes=None, for_training=False) mod.init_params(arg_params=arg_params, aux_params=aux_params) class OneDataBatch(): def __init__(self, img): im_info = mx.nd.array([[img.shape[0], img.shape[1], 1.0]]) img = np.transpose(img, (2, 0, 1)) img = img[np.newaxis, (2, 1, 0)] self.data = [mx.nd.array(img), im_info] self.label = None self.provide_label = None self.provide_data = [("data", (1, 3, img.shape[2], img.shape[3])), ("im_info", (1, 3))] imglist_file = os.path.join(default.dataset_path, 'imglists', 'test.lst') #print(default.dataset_path) assert os.path.exists(imglist_file), 'Path does not exist: {}'.format( imglist_file) imgfiles_list = [] with open(imglist_file, 'r') as f: for line in f: file_list = dict() label = line.strip().split('\t') file_list['img_path'] = label[1] imgfiles_list.append(file_list) roidb = [] index = 0 submit_dir = os.path.join(default.dataset_path, 'submit') if not os.path.exists(submit_dir): os.makedirs(submit_dir) img_dir = os.path.join(default.dataset_path, 'test_result_img') if not os.path.exists(img_dir): os.makedirs(img_dir) for im in range(len(imgfiles_list)): index = im + 1 img_path = os.path.join(default.dataset_path, 'ch4_test_images', 'img_' + str(index) + '.jpg') img_ori = cv2.imread(img_path) batch = OneDataBatch(img_ori) mod.forward(batch, False) results = mod.get_outputs() output = dict(zip(mod.output_names, results)) rois = output['rois_output'].asnumpy()[:, 1:] scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0], img_ori.shape[1]]) nms = py_nms_wrapper(config.TEST.NMS) boxes = pred_boxes CLASSES = ('__background__', 'text') all_boxes = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] all_masks = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] label = np.argmax(scores, axis=1) label = label[:, np.newaxis] for cls in CLASSES: cls_ind = CLASSES.index(cls) cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] cls_masks = mask_output[:, cls_ind, :, :] cls_scores = scores[:, cls_ind, np.newaxis] #print cls_scores.shape, label.shape keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0] cls_masks = cls_masks[keep, :, :] dets = np.hstack( (cls_boxes, cls_scores)).astype(np.float32)[keep, :] keep_la = nms(dets) print('------------------------keep_la', keep_la) all_boxes[cls_ind] = dets[keep_la, :] all_masks[cls_ind] = cls_masks[keep_la, :, :] boxes_this_image = [[] ] + [all_boxes[j] for j in range(1, len(CLASSES))] masks_this_image = [[] ] + [all_masks[j] for j in range(1, len(CLASSES))] import copy import random class_names = CLASSES color_white = (255, 255, 255) scale = 1.0 im = copy.copy(img_ori) num_box = 1 num_boxes = 0 mini_box = np.zeros((4, 2)) mini_box = np.int32(mini_box) if (len(dets) == 0): submit_path = os.path.join(submit_dir, 'res_img_{}.txt'.format(index)) result_txt = open(submit_path, 'a') for i in range(0, 4): result_txt.write(str(mini_box[i][0])) result_txt.write(',') result_txt.write(str(mini_box[i][1])) if i < 3: result_txt.write(',') result_txt.write('\r\n') result_txt.close() for k, name in enumerate(class_names): if name == '__background__': continue color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color dets = boxes_this_image[k] masks = masks_this_image[k] #im_binary_merge = np.zeros(im[:,:,0].shape) print('------------------------len(dets)', len(dets)) for i in range(len(dets)): bbox_i = dets[i, :4] * scale #if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] : if bbox_i[2] == bbox_i[0] or bbox_i[3] == bbox_i[1]: continue score_i = dets[i, -1] bbox_i = map(int, bbox_i) mask_i = masks[i, :, :] mask_i = masks[i, :, :] mask_i = cv2.resize(mask_i, (bbox_i[2] - bbox_i[0], (bbox_i[3] - bbox_i[1])), interpolation=cv2.INTER_LINEAR) mask_i[mask_i > 0.3] = 1 mask_i[mask_i <= 0.3] = 0 im_binary_i = np.zeros(im[:, :, 0].shape) im_binary_i[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2]] = im_binary_i[ bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2]] + mask_i #print("len(dets is )-------------------------",len(dets)) overlap = [] overlap_other = [] for j in range(len(dets)): if i == j: continue bbox_j = dets[j, :4] * scale #if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] : if bbox_j[2] == bbox_j[0] or bbox_j[3] == bbox_j[1]: continue num_box += 1 score_j = dets[j, -1] bbox_j = map(int, bbox_j) mask_j = masks[j, :, :] mask_j = masks[j, :, :] mask_j = cv2.resize(mask_j, (bbox_j[2] - bbox_j[0], (bbox_j[3] - bbox_j[1])), interpolation=cv2.INTER_LINEAR) #print("mask_j,score_j,img_path------------------------",mask_j,score_j,img_path) mask_j[mask_j > 0.3] = 1 mask_j[mask_j <= 0.3] = 0 im_binary_j = np.zeros(im[:, :, 0].shape) im_binary_j[bbox_j[1]:bbox_j[3], bbox_j[0]:bbox_j[2]] = im_binary_j[ bbox_j[1]:bbox_j[3], bbox_j[0]:bbox_j[2]] + mask_j im_binary = im_binary_i + im_binary_j #mask_inter = mask_i+mask_j ni = np.sum(im_binary_i == 1) nj = np.sum(im_binary_j == 1) nij = np.sum(im_binary == 2) IOU_ratio = float(nij) / (ni + nj - nij) overlap.append(IOU_ratio) #if np.sum(im_binary_i == 1) == 0: # continue #if np.sum(im_binary_j == 1) == 0: # continue IOU_ratio_self = float( np.sum(im_binary == 2)) / np.sum(im_binary_i == 1) overlap_other.append(IOU_ratio_self) #IOU_ratio_other = float(np.sum(im_binary == 2)) / np.sum(im_binary_j == 1) #overlap_other.append(IOU_ratio_other) if num_box == 1: overlap.append(0) overlap_other.append(0) if np.max(overlap) < 0.6 and split == False and np.max( overlap_other) < 0.9: num_boxes += 1 #cv2.rectangle(im, (bbox_i[0], bbox_i[1]), (bbox_i[2], bbox_i[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[k], score_i), (bbox_i[0], bbox_i[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) px = np.where(mask_i == 1) x_min = np.min(px[1]) y_min = np.min(px[0]) x_max = np.max(px[1]) y_max = np.max(px[0]) if x_max - x_min <= 1 or y_max - y_min <= 1: continue mask_color = random.randint(0, 255) c = random.randint(0, 2) mini_boxt = np.zeros((4, 2)) target = im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2], c] + mask_color * mask_i target[target >= 255] = 255 im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2], c] = target mini_box = minimum_bounding_rectangle(im_binary_i) mini_boxt[0][0] = mini_box[0][1] mini_boxt[0][1] = mini_box[0][0] mini_boxt[1][0] = mini_box[1][1] mini_boxt[1][1] = mini_box[1][0] mini_boxt[2][0] = mini_box[2][1] mini_boxt[2][1] = mini_box[2][0] mini_boxt[3][0] = mini_box[3][1] mini_boxt[3][1] = mini_box[3][0] mini_box = mini_boxt mini_box = np.int32(mini_box) #print("---------------",mini_box) cv2.polylines(im, [mini_box], 1, (255, 255, 255)) submit_path = os.path.join(submit_dir, 'res_img_{}.txt'.format(index)) result_txt = open(submit_path, 'a') for i in range(0, 4): result_txt.write(str(mini_box[i][0])) result_txt.write(',') result_txt.write(str(mini_box[i][1])) if i < 3: result_txt.write(',') result_txt.write('\r\n') result_txt.close() if split == True: if np.max(overlap_other) > 0.6: W = bbox_j[2] - bbox_j[0] H = bbox_j[3] - bbox_j[1] bbox_i[2] = bbox_i[2] - W bbox_i[3] = bbox_i[3] - H num_boxes += 1 cv2.rectangle(im, (bbox_i[0], bbox_i[1]), (bbox_i[2], bbox_i[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[k], score_i), (bbox_i[0], bbox_i[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) px = np.where(mask_i == 1) x_min = np.min(px[1]) y_min = np.min(px[0]) x_max = np.max(px[1]) y_max = np.max(px[0]) if x_max - x_min <= 1 or y_max - y_min <= 1: continue mask_color = random.randint(0, 255) c = random.randint(0, 2) target = im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2], c] + mask_color * mask_i target[target >= 255] = 255 im[bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2], c] = target #inst_path = os.path.join(inst_dir,'result_{}_{}.mat'.format(index,num_boxes)) #io.savemat(inst_path, {'Segmentation': im_binary_i}) #numbox = open('data/boxnum.txt','a') #numbox.write(str(num_boxes)+'\n') #numbox.close() result_img_path = os.path.join(img_dir, 'result_{}.jpg'.format(index)) cv2.imwrite(result_img_path, im) #zip_submit_dir = 'script_test_ch4' zip_file = os.path.join('script_test_ch4', 'submit.zip') createZip(submit_dir, zip_file) os.system( "python ./script_test_ch4/script.py -g=./script_test_ch4/gt.zip -s=./script_test_ch4/submit.zip" )
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): #print('in_detect', threshold, scales, do_flip, do_nms) #print('img_shape: ',img.shape) proposals_list = [] scores_list = [] landmarks_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() im = im[:, :, np.newaxis] if flip: im = im[:, ::-1, :] if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 1), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X1 uses', diff.total_seconds(), 'seconds') #self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) #im_info = [im.shape[0], im.shape[1], im_scale] im_info = [im.shape[0], im.shape[1]] im_tensor = np.zeros((1, 1, im.shape[0], im.shape[1])) #print('im_tensor_shape: ', im_tensor.shape) #print('im_shape: ', im.shape) for i in range(1): im_tensor[ 0, i, :, :] = (im[:, :, i] / self.pixel_scale - self.pixel_means[i]) / self.pixel_stds[i] if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X2 uses', diff.total_seconds(), 'seconds') data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('X3 uses', diff.total_seconds(), 'seconds') self.model.forward(db, is_train=False) net_out = self.model.get_outputs() #print('Len:out: ', len(net_out)) #print('out1: ', net_out[0]) #print('outbbox: ', net_out[1]) #post_nms_topN = self._rpn_post_nms_top_n #min_size_dict = self._rpn_min_size_fpn for _idx, s in enumerate(self._feat_stride_fpn): #if len(scales)>1 and s==32 and im_scale==scales[-1]: # continue _key = 'stride%s' % s stride = int(s) #if self.vote and stride==4 and len(scales)>2 and (im_scale==scales[0]): # continue if self.use_landmarks: idx = _idx * 3 else: idx = _idx * 2 #print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('A uses', diff.total_seconds(), 'seconds') #print(scores.shape) #print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] idx += 1 bbox_deltas = net_out[idx].asnumpy() #if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) #_height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] #print('A: ',A) K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) #print('num_anchors', self._num_anchors['stride%s'%s], file=sys.stderr) #print('HW', (height, width), file=sys.stderr) #print('anchors_fpn', anchors_fpn.shape, file=sys.stderr) #print('anchors', anchors.shape, file=sys.stderr) #print('bbox_deltas', bbox_deltas.shape, file=sys.stderr) #print('scores', scores.shape, file=sys.stderr) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) #print('bbox_deltas.shape[3]:',bbox_deltas.shape[3]) bbox_pred_len = bbox_deltas.shape[3] // A #print('bbox_deltas.shape:',bbox_deltas.shape) #print('boxlen:',bbox_pred_len) bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) #if self.vote: # if im_scale>1.0: # keep = self._filter_boxes2(proposals, 160*im_scale, -1) # else: # keep = self._filter_boxes2(proposals, -1, 100*im_scale) # if stride==4: # keep = self._filter_boxes2(proposals, 12*im_scale, -1) # proposals = proposals[keep, :] # scores = scores[keep] #keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) #proposals = proposals[keep, :] #scores = scores[keep] #print('333', proposals.shape) scores_ravel = scores.ravel() #print('__shapes', proposals.shape, scores_ravel.shape) #print('max score', np.max(scores_ravel)) order = np.where(scores_ravel >= threshold)[0] #print('order:',order) #print('score_order:',scores_ravel[order]) #_scores = scores_ravel[order] #_order = _scores.argsort()[::-1] #order = order[_order] #order = [50] proposals = proposals[order, :] scores = scores[order] if stride == 4 and self.decay4 < 1.0: scores *= self.decay4 if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: idx += 1 landmark_deltas = net_out[idx].asnumpy() landmark_deltas = self._clip_pad( landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) #print(landmark_deltas.shape, landmark_deltas) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 #for a in range(5): # oldx1 = landmarks[:, a].copy() # landmarks[:,a] = im.shape[1] - oldx1 - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] #flandmarks[:, idx*2] = landmarks[:,a*2] #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale #landmarks /= im_scale #landmarks = landmarks.reshape( (-1, landmark_pred_len) ) landmarks_list.append(landmarks) #proposals = np.hstack((proposals, landmarks)) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('B uses', diff.total_seconds(), 'seconds') proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) #print('shapes', proposals.shape, scores.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] #if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] proposals = proposals[order, :] scores = scores[order] if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: #print('pre_det_type: ', type(pre_det)) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) #if self.use_landmarks: # det = np.hstack((det, landmarks)) if self.debug: timeb = datetime.datetime.now() diff = timeb - timea print('C uses', diff.total_seconds(), 'seconds') return det, scores #landmarks
def detect(self, img, threshold=0.5): """ Detect all the faces and landmarks in an image :param img: input image :param threshold: detection threshold :return: tuple faces, landmarks """ proposals_list = [] scores_list = [] landmarks_list = [] im_tensor, im_info, im_scale = self._preprocess_image(img) net_out = self.model(im_tensor) net_out = [elt.numpy() for elt in net_out] sym_idx = 0 for _idx, s in enumerate(self._feat_stride_fpn): _key = 'stride%s' % s scores = net_out[sym_idx] scores = scores[:, :, :, self._num_anchors['stride%s' % s]:] bbox_deltas = net_out[sym_idx + 1] height, width = bbox_deltas.shape[1], bbox_deltas.shape[2] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, s, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = scores.reshape((-1, 1)) bbox_deltas = bbox_deltas bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * self.bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * self.bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * self.bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * self.bbox_stds[3] proposals = self.bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) if s == 4 and self.decay4 < 1.0: scores *= self.decay4 scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] proposals[:, 0:4] /= im_scale proposals_list.append(proposals) scores_list.append(scores) landmark_deltas = net_out[sym_idx + 2] landmark_pred_len = landmark_deltas.shape[3] // A landmark_deltas = landmark_deltas.reshape( (-1, 5, landmark_pred_len // 5)) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) sym_idx += 3 proposals = np.vstack(proposals_list) if proposals.shape[0] == 0: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) keep = self.nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] landmarks = landmarks[keep] return det, landmarks
def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0][0].context.device_id) #nms = cpu_nms_wrapper(self._threshold) cls_prob_dict = dict(zip(self.fpn_keys, in_data[0:len(self.fpn_keys)])) bbox_pred_dict = dict(zip(self.fpn_keys, in_data[len(self.fpn_keys):2*len(self.fpn_keys)])) #for i in xrange(6): # print(i, in_data[i].asnumpy().shape) batch_size = in_data[0].shape[0] if batch_size > 1: raise ValueError("Sorry, multiple images each device is not implemented") pre_nms_topN = self._rpn_pre_nms_top_n post_nms_topN = self._rpn_post_nms_top_n min_size_dict = self._rpn_min_size_fpn proposals_list = [] scores_list = [] for s in self._feat_stride_fpn: stride = int(s) scores = cls_prob_dict['stride%s'%s].asnumpy()[:, self._num_anchors['stride%s'%s]:, :, :] bbox_deltas = bbox_pred_dict['stride%s'%s].asnumpy() im_info = in_data[-1].asnumpy()[0, :] if DEBUG: print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) print 'scale: {}'.format(im_info[2]) #height, width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s'%s] K = height * width anchors = anchors_plane(height, width, stride, self._anchors_fpn['stride%s'%s].astype(np.float32)) anchors = anchors.reshape((K * A, 4)) #print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) #print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape((-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #print(anchors.shape, bbox_deltas.shape, A, K) proposals = self._bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) proposals = proposals[keep, :] scores = scores[keep] proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) order = scores.ravel().argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) if np.shape(det)[0] == 0: print "Something wrong with the input image(resolution is too low?), generate fake proposals for it." proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) keep = nms(det) if post_nms_topN > 0: keep = keep[:post_nms_topN] if len(keep) < post_nms_topN: pad = npr.choice(keep, size=post_nms_topN - len(keep)) keep = np.hstack((keep, pad)) proposals = proposals[keep, :] scores = scores[keep] batch_inds = np.zeros((proposals.shape[0], 1), dtype=np.float32) blob = np.hstack((batch_inds, proposals.astype(np.float32, copy=False))) self.assign(out_data[0], req[0], blob) if self._output_score: self.assign(out_data[1], req[1], scores.astype(np.float32, copy=False))
def postprocess(net_out, threshold, ctx_id, im_scale, im_info): # im_info = [640, 640] flip = False decay4 = 0.5 vote = False fpn_keys = [] anchor_cfg = None bbox_stds = [1.0, 1.0, 1.0, 1.0] # im_scale = 1.0 landmark_std = 1.0 nms_threshold = 0.4 proposals_list = [] scores_list = [] landmarks_list = [] strides_list = [] use_landmarks = True if ctx_id >= 0: nms = gpu_nms_wrapper(nms_threshold, ctx_id) else: nms = cpu_nms_wrapper(nms_threshold) use_landmarks = True _ratio = (1., ) _feat_stride_fpn = [32, 16, 8] anchor_cfg = { '32': { 'SCALES': (32, 16), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999 }, '16': { 'SCALES': (8, 4), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999 }, '8': { 'SCALES': (2, 1), 'BASE_SIZE': 16, 'RATIOS': _ratio, 'ALLOWED_BORDER': 9999 }, } for s in _feat_stride_fpn: fpn_keys.append('stride%s' % s) dense_anchor = False _anchors_fpn = dict( zip(fpn_keys, generate_anchors_fpn(dense_anchor=dense_anchor, cfg=anchor_cfg))) for k in _anchors_fpn: v = _anchors_fpn[k].astype(np.float32) _anchors_fpn[k] = v _num_anchors = dict( zip(fpn_keys, [anchors.shape[0] for anchors in _anchors_fpn.values()])) sym_idx = 0 for _idx, s in enumerate(_feat_stride_fpn): # print(sym_idx) _key = 'stride%s' % s # print(_key) stride = int(s) scores = net_out[sym_idx] #.asnumpy() scores = scores[:, _num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[sym_idx + 1] # .asnumpy() height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = _num_anchors['stride%s' % s] K = height * width anchors_fpn = _anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) anchors = anchors.reshape((K * A, 4)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)) bbox_pred_len = bbox_deltas.shape[3] // A bbox_deltas = bbox_deltas.reshape((-1, bbox_pred_len)) bbox_deltas[:, 0::4] = bbox_deltas[:, 0::4] * bbox_stds[0] bbox_deltas[:, 1::4] = bbox_deltas[:, 1::4] * bbox_stds[1] bbox_deltas[:, 2::4] = bbox_deltas[:, 2::4] * bbox_stds[2] bbox_deltas[:, 3::4] = bbox_deltas[:, 3::4] * bbox_stds[3] proposals = bbox_pred(anchors, bbox_deltas) proposals = clip_boxes(proposals, im_info[:2]) if stride == 4 and decay4 < 1.0: scores *= decay4 scores_ravel = scores.ravel() order = np.where(scores_ravel >= threshold)[0] proposals = proposals[order, :] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 #proposals[:,0:4] /= im_scale #print(proposals[:,0]) proposals[:, 0] /= im_scale[0] #print(pp) proposals[:, 1] /= im_scale[1] proposals[:, 2] /= im_scale[0] proposals[:, 3] /= im_scale[1] #print(proposals[:,0]) proposals_list.append(proposals) scores_list.append(scores) if nms_threshold < 0.0: _strides = np.empty(shape=(scores.shape), dtype=np.float32) _strides.fill(stride) strides_list.append(_strides) if not vote and use_landmarks: landmark_deltas = net_out[sym_idx + 2] #.asnumpy() # print(landmark_deltas) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) landmark_deltas *= landmark_std landmarks = landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) if use_landmarks: sym_idx += 3 else: sym_idx += 2 proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if use_landmarks: landmarks = np.zeros((0, 5, 2)) if nms_threshold < 0.0: return np.zeros((0, 6)), landmarks else: return np.zeros((0, 5)), landmarks scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] proposals = proposals[order, :] scores = scores[order] if nms_threshold < 0.0: strides = np.vstack(strides_list) strides = strides[order] if not vote and use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) if nms_threshold > 0.0: pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not vote: keep = nms(pre_det) det = np.hstack((pre_det, proposals[:, 4:])) det = det[keep, :] if use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = bbox_vote(det, nms_threshold) elif nms_threshold < 0.0: det = np.hstack((proposals[:, 0:4], scores, strides)).astype(np.float32, copy=False) else: det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) return det, landmarks
def detect(self, img, threshold=0.05, scales=[1.0]): proposals_list = [] scores_list = [] for im_scale in scales: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img im = im.astype(np.float32) # self.model.bind(data_shapes=[('data', (1, 3, image_size[0], image_size[1]))], for_training=False) im_info = [im.shape[0], im.shape[1], im_scale] im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = im[:, :, 2 - i] - self.pixel_means[2 - i] data = nd.array(im_tensor) db = mx.io.DataBatch(data=(data, ), provide_data=[('data', data.shape)]) self.model.forward(db, is_train=False) net_out = self.model.get_outputs() pre_nms_topN = self._rpn_pre_nms_top_n # post_nms_topN = self._rpn_post_nms_top_n # min_size_dict = self._rpn_min_size_fpn for s in self._feat_stride_fpn: if len(scales) > 1 and s == 32 and im_scale == scales[-1]: continue _key = 'stride%s' % s stride = int(s) idx = 0 if s == 16: idx = 2 elif s == 8: idx = 4 print('getting', im_scale, stride, idx, len(net_out), data.shape, file=sys.stderr) scores = net_out[idx].asnumpy() # print(scores.shape) idx += 1 # print('scores',stride, scores.shape, file=sys.stderr) scores = scores[:, self._num_anchors['stride%s' % s]:, :, :] bbox_deltas = net_out[idx].asnumpy() # if DEBUG: # print 'im_size: ({}, {})'.format(im_info[0], im_info[1]) # print 'scale: {}'.format(im_info[2]) _height, _width = int(im_info[0] / stride), int(im_info[1] / stride) height, width = bbox_deltas.shape[2], bbox_deltas.shape[3] A = self._num_anchors['stride%s' % s] K = height * width anchors = anchors_plane( height, width, stride, self._anchors_fpn['stride%s' % s].astype(np.float32)) # print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) # print('pre', bbox_deltas.shape, height, width) bbox_deltas = self._clip_pad(bbox_deltas, (height, width)) # print('after', bbox_deltas.shape, height, width) bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1)).reshape( (-1, 4)) scores = self._clip_pad(scores, (height, width)) scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) # print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self._bbox_pred(anchors, bbox_deltas) # proposals = anchors proposals = clip_boxes(proposals, im_info[:2]) # keep = self._filter_boxes(proposals, min_size_dict['stride%s'%s] * im_info[2]) # proposals = proposals[keep, :] # scores = scores[keep] # print('333', proposals.shape) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] if pre_nms_topN > 0: order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] proposals /= im_scale proposals_list.append(proposals) scores_list.append(scores) proposals = np.vstack(proposals_list) scores = np.vstack(scores_list) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] # if config.TEST.SCORE_THRESH>0.0: # _count = np.sum(scores_ravel>config.TEST.SCORE_THRESH) # order = order[:_count] # if pre_nms_topN > 0: # order = order[:pre_nms_topN] proposals = proposals[order, :] scores = scores[order] det = np.hstack((proposals, scores)).astype(np.float32) # if np.shape(det)[0] == 0: # print("Something wrong with the input image(resolution is too low?), generate fake proposals for it.") # proposals = np.array([[1.0, 1.0, 2.0, 2.0]]*post_nms_topN, dtype=np.float32) # scores = np.array([[0.9]]*post_nms_topN, dtype=np.float32) # det = np.array([[1.0, 1.0, 2.0, 2.0, 0.9]]*post_nms_topN, dtype=np.float32) if self.nms_threshold < 1.0: keep = self.nms(det) det = det[keep, :] if threshold > 0.0: keep = np.where(det[:, 4] >= threshold)[0] det = det[keep, :] return det
def detect(self, img, threshold=0.5, scales=[1.0], do_flip=False): print( 'get into detect, confi thresold={}, scales={}, do_flip={}'.format( threshold, scales, do_flip)) proposals_list = [] scores_list = [] landmarks_list = [] timea = datetime.datetime.now() flips = [0] if do_flip: flips = [0, 1] #TODO 根据scale给输入的图片做resize for im_scale in scales: for flip in flips: if im_scale != 1.0: im = cv2.resize(img, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) else: im = img.copy() # 对图像做翻转 if flip: im = im[:, ::-1, :] # 对图像做裁剪 if self.nocrop: if im.shape[0] % 32 == 0: h = im.shape[0] else: h = (im.shape[0] // 32 + 1) * 32 if im.shape[1] % 32 == 0: w = im.shape[1] else: w = (im.shape[1] // 32 + 1) * 32 _im = np.zeros((h, w, 3), dtype=np.float32) _im[0:im.shape[0], 0:im.shape[1], :] = im im = _im else: im = im.astype(np.float32) im_info = [im.shape[0], im.shape[1]] #h,w im_tensor = np.zeros((1, 3, im.shape[0], im.shape[1])) for i in range(3): im_tensor[0, i, :, :] = ( im[:, :, 2 - i] / self.pixel_scale - self.pixel_means[2 - i]) / self.pixel_stds[ 2 - i] #TODO 这里好像将Channel顺序倒过来了,与image.py保持一致 data = np.array(im_tensor) # 读入模型进行推理,得到预测值 net_out = self.get_pred(data) if self.debug: for key in net_out.keys(): print('{} = {}\n'.format(key, net_out[key].shape)) for _idx, s in enumerate(self._feat_stride_fpn): # print('begin stride{}-------------------------------------------------\n'.format(s)) _key = 'stride%s' % s stride = int(s) # print('getting im_scale={}, stride={}, len(net_out)={}, data.shape={}'.format(im_scale, stride, len(net_out), data.shape)) scores = net_out['rpn_cls_prob_stride%s' % s] #TODO 要注意这里是nhwc不是nchw if self.debug: print('get score:', scores.shape) # print('stride{}: scores before shape={}, idx={}'.format(stride, scores.shape, self._num_anchors['stride%s' % s])) scores = scores[:, 1].reshape( (-1, 1)) #TODO: (H*W*A, 1) #这里的1表示正类的概率 if self.debug: print('AAAAstride{}: scores after shape={}'.format( stride, scores.shape)) bbox_deltas = net_out['rpn_bbox_pred_stride%s' % s] #TODO NHW8 height, width = bbox_deltas.shape[1], bbox_deltas.shape[2] A = self._num_anchors['stride%s' % s] K = height * width anchors_fpn = self._anchors_fpn['stride%s' % s] anchors = anchors_plane(height, width, stride, anchors_fpn) #获取该特征图上的所有anchor #print((height, width), (_height, _width), anchors.shape, bbox_deltas.shape, scores.shape, file=sys.stderr) anchors = anchors.reshape((K * A, 4)) if self.debug: print('HW', (height, width)) print('anchors_fpn', anchors_fpn) print('anchors', anchors.shape, '\n') # scores = self._clip_pad_NCHW(scores, (height, width)) #(1, 4, H, W) # scores = scores.transpose((0, 2, 3, 1)).reshape((-1, 1)) #(1, H, W, 4) # print('scores reshape', scores.shape) if self.debug: print('before bbox_deltas', bbox_deltas.shape) bbox_deltas = self._clip_pad_NHWC( bbox_deltas, (height, width)) #(1, H, W, 8) # bbox_deltas = bbox_deltas.transpose((0, 2, 3, 1))#(1, H, W, 8) bbox_pred_len = bbox_deltas.shape[3] // A #4 bbox_deltas = bbox_deltas.reshape( (-1, bbox_pred_len)) #(H*W*2, 4) if self.debug: print('after bbox_deltas', bbox_deltas.shape, height, width, '\n') #print(anchors.shape, bbox_deltas.shape, A, K, file=sys.stderr) proposals = self.bbox_pred( anchors, bbox_deltas) #TODO important! 将anchor加上delta进行处理 proposals = clip_boxes(proposals, im_info[:2]) scores_ravel = scores.ravel() max_score = np.max(scores_ravel) print('proposals.shape={}, score_ravel.shape={}'.format( proposals.shape, scores_ravel.shape)) print('max score', max_score) order = np.where(scores_ravel >= threshold)[0] #_scores = scores_ravel[order] #_order = _scores.argsort()[::-1] #order = order[_order] proposals = proposals[order, :] scores = scores[order] if flip: oldx1 = proposals[:, 0].copy() oldx2 = proposals[:, 2].copy() proposals[:, 0] = im.shape[1] - oldx2 - 1 proposals[:, 2] = im.shape[1] - oldx1 - 1 proposals[:, 0: 4] /= im_scale #TODO important 在这里将找到的proposal给映射回原来图像的位置 proposals_list.append(proposals) scores_list.append(scores) if not self.vote and self.use_landmarks: landmark_deltas = net_out['rpn_landmark_pred_stride%s' % s] #(1,20,H,W) if self.debug: print('before landmark_deltas', landmark_deltas.shape) landmark_deltas = self._clip_pad_NCHW( landmark_deltas, (height, width)) landmark_pred_len = landmark_deltas.shape[1] // A landmark_deltas = landmark_deltas.transpose( (0, 2, 3, 1)).reshape( (-1, 5, landmark_pred_len // 5)) if self.debug: print('after landmark_deltas', landmark_deltas.shape, landmark_deltas) landmarks = self.landmark_pred(anchors, landmark_deltas) landmarks = landmarks[order, :] if flip: landmarks[:, :, 0] = im.shape[1] - landmarks[:, :, 0] - 1 #for a in range(5): # oldx1 = landmarks[:, a].copy() # landmarks[:,a] = im.shape[1] - oldx1 - 1 order = [1, 0, 2, 4, 3] flandmarks = landmarks.copy() for idx, a in enumerate(order): flandmarks[:, idx, :] = landmarks[:, a, :] #flandmarks[:, idx*2] = landmarks[:,a*2] #flandmarks[:, idx*2+1] = landmarks[:,a*2+1] landmarks = flandmarks landmarks[:, :, 0:2] /= im_scale landmarks_list.append(landmarks) if self.debug: print( 'end stride{}-------------------------------------------------\n' .format(s)) proposals = np.vstack(proposals_list) landmarks = None if proposals.shape[0] == 0: if self.use_landmarks: landmarks = np.zeros((0, 5, 2)) return np.zeros((0, 5)), landmarks # for i in range(len(scores_list)): # print('hhhhh score,shape=',scores_list[i].shape) scores = np.vstack(scores_list) print('finally!!! proposals.shape={}, score.shape={}'.format( proposals.shape, scores.shape)) scores_ravel = scores.ravel() order = scores_ravel.argsort()[::-1] # 按照score从大到小排序 proposals = proposals[order, :] scores = scores[order] if self.debug: print('sort score=', scores) if not self.vote and self.use_landmarks: landmarks = np.vstack(landmarks_list) landmarks = landmarks[order].astype(np.float32, copy=False) pre_det = np.hstack((proposals[:, 0:4], scores)).astype(np.float32, copy=False) if not self.vote: print('begin to NMS!!\n') keep = self.nms(pre_det) # print('before hstack: pre_det={}, proposals.shape={}, proposals[:,4:]={}'.format(pre_det.shape, proposals.shape, proposals[:,4:])) det = np.hstack((pre_det, proposals[:, 4:])) # print('after hstack: pre_det={}, proposals.shape={}'.format(pre_det.shape, proposals.shape)) det = det[keep, :] if self.use_landmarks: landmarks = landmarks[keep] else: det = np.hstack((pre_det, proposals[:, 4:])) det = self.bbox_vote(det) return det, landmarks
def demo_maskrcnn(network, ctx, prefix, epoch, img_path, vis=True, has_rpn=True, thresh=0.001): assert has_rpn, "Only has_rpn==True has been supported." #sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS) sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES) arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True) for k, v in arg_params.iteritems(): print(k, v.shape) max_image_shape = (1, 3, 1024, 1024) max_data_shapes = [("data", max_image_shape), ("im_info", (1, 3))] mod = MutableModule(symbol=sym, data_names=["data", "im_info"], label_names=None, max_data_shapes=max_data_shapes, context=ctx) mod.bind(data_shapes=max_data_shapes, label_shapes=None, for_training=False) mod.init_params(arg_params=arg_params, aux_params=aux_params) class OneDataBatch(): def __init__(self, img): im_info = mx.nd.array([[img.shape[0], img.shape[1], 1.0]]) img = np.transpose(img, (2, 0, 1)) img = img[np.newaxis, (2, 1, 0)] self.data = [mx.nd.array(img), im_info] self.label = None self.provide_label = None self.provide_data = [("data", (1, 3, img.shape[2], img.shape[3])), ("im_info", (1, 3))] img_ori = cv2.imread(img_path) batch = OneDataBatch(img_ori) mod.forward(batch, False) results = mod.get_outputs() output = dict(zip(mod.output_names, results)) rois = output['rois_output'].asnumpy()[:, 1:] scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0], img_ori.shape[1]]) nms = py_nms_wrapper(config.TEST.NMS) #nms = processing_nms_wrapper(config.TEST.NMS, 0.7) boxes = pred_boxes CLASSES = ('__background__', 'text') all_boxes = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] all_masks = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] label = np.argmax(scores, axis=1) label = label[:, np.newaxis] for cls in CLASSES: cls_ind = CLASSES.index(cls) cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] cls_masks = mask_output[:, cls_ind, :, :] cls_scores = scores[:, cls_ind, np.newaxis] #print cls_scores.shape, label.shape keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0] cls_masks = cls_masks[keep, :, :] dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :] keep = nms(dets) #print dets.shape, cls_masks.shape all_boxes[cls_ind] = dets[keep, :] all_masks[cls_ind] = cls_masks[keep, :, :] boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))] masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))] import copy import random class_names = CLASSES color_white = (255, 255, 255) scale = 1.0 im = copy.copy(img_ori) for j, name in enumerate(class_names): if name == '__background__': continue color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color dets = boxes_this_image[j] masks = masks_this_image[j] for i in range(len(dets)): bbox = dets[i, :4] * scale if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[ 1] or bbox[2] == bbox[3]: continue score = dets[i, -1] bbox = map(int, bbox) cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) mask = masks[i, :, :] mask = cv2.resize(mask, (bbox[2] - bbox[0], (bbox[3] - bbox[1])), interpolation=cv2.INTER_LINEAR) mask[mask > 0.5] = 1 mask[mask <= 0.5] = 0 mask_color = random.randint(0, 255) c = random.randint(0, 2) target = im[bbox[1]:bbox[3], bbox[0]:bbox[2], c] + mask_color * mask target[target >= 255] = 255 im[bbox[1]:bbox[3], bbox[0]:bbox[2], c] = target ##im = im[:,:,(2,1,0)] ##plt.imshow(im) cv2.imwrite("figures/test_result.jpg", im)
def demo_maskrcnn(network, ctx, prefix, epoch, vis=True, has_rpn=True, thresh=0.001): assert has_rpn, "Only has_rpn==True has been supported." sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES) arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True) max_image_shape = (1, 3, 1024, 1024) max_data_shapes = [("data", max_image_shape), ("im_info", (1, 3))] mod = MutableModule(symbol=sym, data_names=["data", "im_info"], label_names=None, max_data_shapes=max_data_shapes, context=ctx) mod.bind(data_shapes=max_data_shapes, label_shapes=None, for_training=False) mod.init_params(arg_params=arg_params, aux_params=aux_params) class OneDataBatch(): def __init__(self, img): im_info = mx.nd.array([[img.shape[0], img.shape[1], 1.0]]) img = np.transpose(img, (2, 0, 1)) img = img[np.newaxis, (2, 1, 0)] self.data = [mx.nd.array(img), im_info] self.label = None self.provide_label = None self.provide_data = [("data", (1, 3, img.shape[2], img.shape[3])), ("im_info", (1, 3))] #img_ori = cv2.imread(img_path) #batch = OneDataBatch(img_ori) #mod.forward(batch, False) #results = mod.get_outputs() #output = dict(zip(mod.output_names, results)) #rois = output['rois_output'].asnumpy()[:, 1:] #scores = output['cls_prob_reshape_output'].asnumpy()[0] #bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] #mask_output = output['mask_prob_output'].asnumpy() #pred_boxes = bbox_pred(rois, bbox_deltas) #pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0],img_ori.shape[1]]) #nms = py_nms_wrapper(config.TEST.NMS) #boxes= pred_boxes #CLASSES = ('__background__', 'person', 'rider', 'car', 'truck', 'bus', 'train', 'mcycle', 'bicycle') #CLASSES = ('__background__', 'text') #all_boxes = [[[] for _ in xrange(1)] # for _ in xrange(len(CLASSES))] #all_masks = [[[] for _ in xrange(1)] # for _ in xrange(len(CLASSES))] #label = np.argmax(scores, axis=1) #label = label[:, np.newaxis] #for cls in CLASSES: # cls_ind = CLASSES.index(cls) # cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] # cls_masks = mask_output[:, cls_ind, :, :] # cls_scores = scores[:, cls_ind, np.newaxis] # #print cls_scores.shape, label.shape # keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0] # cls_masks = cls_masks[keep, :, :] # dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :] # keep = nms(dets) # #print dets.shape, cls_masks.shape # all_boxes[cls_ind] = dets[keep, :] # all_masks[cls_ind] = cls_masks[keep, :, :] #boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))] #masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))] #import copy #import random # class_names = CLASSES #color_white = (255, 255, 255) #scale = 1.0 #im = copy.copy(img_ori) #for j, name in enumerate(class_names): # if name == '__background__': # continue # color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color # dets = boxes_this_image[j] # masks = masks_this_image[j] # for i in range(len(dets)): # bbox = dets[i, :4] * scale # if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] : # continue # score = dets[i, -1] # bbox = map(int, bbox) # cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) # cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), # color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) # mask = masks[i, :, :] # mask = cv2.resize(mask, (bbox[2] - bbox[0], (bbox[3] - bbox[1])), interpolation=cv2.INTER_LINEAR) #3 # mask[mask > 0.5] = 1 # mask[mask <= 0.5] = 0 # mask_color = random.randint(0, 255) # c = random.randint(0, 2) # target = im[bbox[1]: bbox[3], bbox[0]: bbox[2], c] + mask_color * mask # target[target >= 255] = 255 # im[bbox[1]: bbox[3], bbox[0]: bbox[2], c] = target #im = im[:,:,(2,1,0)] #cv2.imwrite("figures/test_result.jpg",im) #plt.imshow(im) #fig1 = plt.gcf() #plt.savefig("figures/test_result.jpg") #if vis: #plt.show() #else: imglist_file = os.path.join(default.dataset_path, 'imglists', 'test.lst') assert os.path.exists(imglist_file), 'Path does not exist: {}'.format( imglist_file) imgfiles_list = [] with open(imglist_file, 'r') as f: for line in f: file_list = dict() label = line.strip().split('\t') #file_list['img_id'] = label[0] file_list['img_path'] = label[1] #file_list['ins_seg_path'] = label[2].replace('labelTrainIds', 'instanceIds') imgfiles_list.append(file_list) #assert len(imgfiles_list) == self.num_images, 'number of boxes matrix must match number of images' roidb = [] index = 0 for im in range(len(imgfiles_list)): #print '===============================', im, '=====================================' #roi_rec = dict() #img_path = os.path.join(self.data_path, imgfiles_list[im]['img_path']) index = im + 1 img_path = os.path.join(default.dataset_path, 'ch4_test_images', 'img_' + str(index) + '.jpg') #size = cv2.imread(roi_rec['image']).shape #roi_rec['height'] = size[0] #roi_rec['width'] = size[1] #img_path = os.path.join(img_path, 'img_' + index + '.jpg') img_ori = cv2.imread(img_path) #img_ori = cv2.resize(img_ori, (, 28), interpolation=cv2.INTER_NEAREST) batch = OneDataBatch(img_ori) mod.forward(batch, False) results = mod.get_outputs() output = dict(zip(mod.output_names, results)) rois = output['rois_output'].asnumpy()[:, 1:] scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0], img_ori.shape[1]]) #nms = py_nms_wrapper(config.TEST.NMS) nms = processing_nms_wrapper(config.TEST.NMS, 0.8) boxes = pred_boxes CLASSES = ('__background__', 'text') all_boxes = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] all_masks = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] label = np.argmax(scores, axis=1) label = label[:, np.newaxis] for cls in CLASSES: cls_ind = CLASSES.index(cls) cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] cls_masks = mask_output[:, cls_ind, :, :] cls_scores = scores[:, cls_ind, np.newaxis] #print cls_scores.shape, label.shape keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0] cls_masks = cls_masks[keep, :, :] dets = np.hstack( (cls_boxes, cls_scores)).astype(np.float32)[keep, :] keep = nms(dets) #print dets.shape, cls_masks.shape all_boxes[cls_ind] = dets[keep, :] all_masks[cls_ind] = cls_masks[keep, :, :] boxes_this_image = [[] ] + [all_boxes[j] for j in range(1, len(CLASSES))] masks_this_image = [[] ] + [all_masks[j] for j in range(1, len(CLASSES))] import copy import random class_names = CLASSES color_white = (255, 255, 255) scale = 1.0 im = copy.copy(img_ori) num_boxes = 0 for j, name in enumerate(class_names): if name == '__background__': continue color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color dets = boxes_this_image[j] masks = masks_this_image[j] for i in range(len(dets)): #num_boxes += 1 bbox = dets[i, :4] * scale #if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] : if bbox[2] == bbox[0] or bbox[3] == bbox[1]: continue num_boxes += 1 score = dets[i, -1] bbox = map(int, bbox) cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) mask = masks[i, :, :] mask = cv2.resize(mask, (bbox[2] - bbox[0], (bbox[3] - bbox[1])), interpolation=cv2.INTER_LINEAR) mask[mask > 0.5] = 1 mask[mask <= 0.5] = 0 px = np.where(mask == 1) x_min = np.min(px[1]) y_min = np.min(px[0]) x_max = np.max(px[1]) y_max = np.max(px[0]) #if x_max - x_min <= 1 or y_max - y_min <= 1: # continue im_binary = np.zeros(im[:, :, 0].shape) im_binary[bbox[1]:bbox[3], bbox[0]:bbox[2]] = im_binary[bbox[1]:bbox[3], bbox[0]:bbox[2]] + mask mask_color = random.randint(0, 255) c = random.randint(0, 2) target = im[bbox[1]:bbox[3], bbox[0]:bbox[2], c] + mask_color * mask target[target >= 255] = 255 im[bbox[1]:bbox[3], bbox[0]:bbox[2], c] = target #cv2.imwrite("figures/test_result.jpg",im) inst_dir = os.path.join(default.dataset_path, 'test_mat') if not os.path.exists(inst_dir): os.makedirs(inst_dir) inst_path = os.path.join( inst_dir, 'result_{}_{}.mat'.format(index, num_boxes)) io.savemat(inst_path, {'Segmentation': im_binary}) numbox = open('data/boxnum.txt', 'a') numbox.write(str(num_boxes) + '\n') numbox.close() img_dir = os.path.join(default.dataset_path, 'test_result_img') if not os.path.exists(img_dir): os.makedirs(img_dir) img_path = os.path.join(img_dir, 'result_{}.jpg'.format(index)) cv2.imwrite(img_path, im)
def demo_maskrcnn(network, ctx, prefix, epoch,img_path, vis= True, has_rpn = True, thresh = 0.001): assert has_rpn,"Only has_rpn==True has been supported." sym = eval('get_' + network + '_mask_test')(num_classes=config.NUM_CLASSES, num_anchors=config.NUM_ANCHORS) arg_params, aux_params = load_param(prefix, epoch, convert=True, ctx=ctx, process=True) max_image_shape = (1,3,1024,1024) max_data_shapes = [("data",max_image_shape),("im_info",(1,3))] mod = MutableModule(symbol = sym, data_names = ["data","im_info"], label_names= None, max_data_shapes = max_data_shapes, context=ctx) mod.bind(data_shapes = max_data_shapes, label_shapes = None, for_training=False) mod.init_params(arg_params=arg_params, aux_params=aux_params) class OneDataBatch(): def __init__(self,img): im_info = mx.nd.array([[img.shape[0],img.shape[1],1.0]]) img = np.transpose(img,(2,0,1)) img = img[np.newaxis,(2,1,0)] self.data = [mx.nd.array(img),im_info] self.label = None self.provide_label = None self.provide_data = [("data",(1,3,img.shape[2],img.shape[3])),("im_info",(1,3))] img_ori = cv2.imread(img_path) batch = OneDataBatch(img_ori) mod.forward(batch, False) results = mod.get_outputs() output = dict(zip(mod.output_names, results)) rois = output['rois_output'].asnumpy()[:, 1:] scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, [img_ori.shape[0],img_ori.shape[1]]) nms = py_nms_wrapper(config.TEST.NMS) boxes= pred_boxes CLASSES = ('__background__', 'person', 'rider', 'car', 'truck', 'bus', 'train', 'mcycle', 'bicycle') all_boxes = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] all_masks = [[[] for _ in xrange(1)] for _ in xrange(len(CLASSES))] label = np.argmax(scores, axis=1) label = label[:, np.newaxis] for cls in CLASSES: cls_ind = CLASSES.index(cls) cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] cls_masks = mask_output[:, cls_ind, :, :] cls_scores = scores[:, cls_ind, np.newaxis] #print cls_scores.shape, label.shape keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0] cls_masks = cls_masks[keep, :, :] dets = np.hstack((cls_boxes, cls_scores)).astype(np.float32)[keep, :] keep = nms(dets) #print dets.shape, cls_masks.shape all_boxes[cls_ind] = dets[keep, :] all_masks[cls_ind] = cls_masks[keep, :, :] boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))] masks_this_image = [[]] + [all_masks[j] for j in range(1, len(CLASSES))] import copy import random class_names = CLASSES color_white = (255, 255, 255) scale = 1.0 im = copy.copy(img_ori) for j, name in enumerate(class_names): if name == '__background__': continue color = (random.randint(0, 256), random.randint(0, 256), random.randint(0, 256)) # generate a random color dets = boxes_this_image[j] masks = masks_this_image[j] for i in range(len(dets)): bbox = dets[i, :4] * scale if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] : continue score = dets[i, -1] bbox = map(int, bbox) cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color=color, thickness=2) cv2.putText(im, '%s %.3f' % (class_names[j], score), (bbox[0], bbox[1] + 10), color=color_white, fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5) mask = masks[i, :, :] mask = cv2.resize(mask, (bbox[2] - bbox[0], (bbox[3] - bbox[1])), interpolation=cv2.INTER_LINEAR) mask[mask > 0.5] = 1 mask[mask <= 0.5] = 0 mask_color = random.randint(0, 255) c = random.randint(0, 2) target = im[bbox[1]: bbox[3], bbox[0]: bbox[2], c] + mask_color * mask target[target >= 255] = 255 im[bbox[1]: bbox[3], bbox[0]: bbox[2], c] = target im = im[:,:,(2,1,0)] plt.imshow(im) if vis: plt.show() else: plt.savefig("figures/test_result.jpg")
def detect(self, img, scales=[1.], thresh=0.5): ret = [] #scale = scales[0] dets_all = None masks_all = None for scale in scales: if scale != 1.0: nimg = cv2.resize(img, None, None, fx=scale, fy=scale, interpolation=cv2.INTER_LINEAR) else: nimg = img im_size = nimg.shape[0:2] #im_info = mx.nd.array([[nimg.shape[0],nimg.shape[1],1.0]]) #nimg = np.transpose(nimg,(2,0,1)) #nimg = nimg[np.newaxis,(2,1,0)] #nimg = mx.nd.array(nimg) #db = mx.io.DataBatch(data=(nimg,im_info)) db = OneDataBatch(nimg) self.model.forward(db, is_train=False) results = self.model.get_outputs() output = dict(zip(self.model.output_names, results)) rois = output['rois_output'].asnumpy()[:, 1:] scores = output['cls_prob_reshape_output'].asnumpy()[0] bbox_deltas = output['bbox_pred_reshape_output'].asnumpy()[0] mask_output = output['mask_prob_output'].asnumpy() pred_boxes = bbox_pred(rois, bbox_deltas) pred_boxes = clip_boxes(pred_boxes, [im_size[0], im_size[1]]) boxes = pred_boxes label = np.argmax(scores, axis=1) label = label[:, np.newaxis] cls_ind = 1 #text class cls_boxes = boxes[:, 4 * cls_ind:4 * (cls_ind + 1)] / scale cls_masks = mask_output[:, cls_ind, :, :] cls_scores = scores[:, cls_ind, np.newaxis] #print cls_scores.shape, label.shape keep = np.where((cls_scores >= thresh) & (label == cls_ind))[0] dets = np.hstack( (cls_boxes, cls_scores)).astype(np.float32)[keep, :] masks = cls_masks[keep, :, :] if dets.shape[0] == 0: continue if dets_all is None: dets_all = dets masks_all = masks else: dets_all = np.vstack((dets_all, dets)) masks_all = np.vstack((masks_all, masks)) #scores = dets[:,4] #index = np.argsort(scores)[::-1] #dets = dets[index] #print(dets) if dets_all is None: return np.zeros((0, 2)) dets = dets_all masks = masks_all keep = self.nms(dets) dets = dets[keep, :] masks = masks[keep, :, :] det_mask = np.zeros((dets.shape[0], ) + img.shape[0:2], dtype=np.int) mask_n = np.zeros((dets.shape[0], ), dtype=np.int) invalid = np.zeros((dets.shape[0], ), dtype=np.int) for i in range(dets.shape[0]): bbox_i = dets[i, :4] #if bbox[2] == bbox[0] or bbox[3] == bbox[1] or bbox[0] == bbox[1] or bbox[2] == bbox[3] : if bbox_i[2] == bbox_i[0] or bbox_i[3] == bbox_i[1]: invalid[i] = 1 continue score_i = dets[i, -1] #bbox_i = map(int, bbox_i) bbox_i = bbox_i.astype(np.int) mask_i = masks[i, :, :] mask_i = cv2.resize(mask_i, (bbox_i[2] - bbox_i[0], (bbox_i[3] - bbox_i[1])), interpolation=cv2.INTER_LINEAR) #avg_mask = np.mean(mask_i[mask_i>0.5]) #print('det', i, 'mask avg', avg_mask) mask_i[mask_i > 0.5] = 1 mask_i[mask_i <= 0.5] = 0 det_mask[i, bbox_i[1]:bbox_i[3], bbox_i[0]:bbox_i[2]] += mask_i.astype(np.int) mask_n[i] = np.sum(mask_i == 1) if self.mask_nms: for i in range(dets.shape[0]): if invalid[i] > 0: continue mask_i = det_mask[i] ni = mask_n[i] merge_list = [] for j in range(i + 1, dets.shape[0]): if invalid[j] > 0: continue mask_j = det_mask[j] nj = mask_n[j] mask_inter = mask_i + mask_j nij = np.sum(mask_inter == 2) iou = float(nij) / (ni + nj - nij) iou_i = float(nij) / ni iou_j = float(nij) / nj if iou_j > 0.7: invalid[j] = 1 if iou >= config.TEST.NMS: #if iou>=0.7: invalid[j] = 1 if iou >= MERGE_THRESH: merge_list.append(j) #mask_i = np.logical_or(mask_i, mask_j, dtype=np.int).astype(np.int) #det_mask[i] = mask_i #print(mask_i) for mm in merge_list: _mask = det_mask[mm] mask_i = np.logical_or(mask_i, _mask, dtype=np.int) if len(merge_list) > 0: det_mask[i] = mask_i.astype(np.int) for i in range(dets.shape[0]): if invalid[i] > 0: continue mask_i = det_mask[i] mini_box = minimum_bounding_rectangle(mask_i) mini_boxt = np.zeros((4, 2)) mini_boxt[0][0] = mini_box[0][1] mini_boxt[0][1] = mini_box[0][0] mini_boxt[1][0] = mini_box[1][1] mini_boxt[1][1] = mini_box[1][0] mini_boxt[2][0] = mini_box[2][1] mini_boxt[2][1] = mini_box[2][0] mini_boxt[3][0] = mini_box[3][1] mini_boxt[3][1] = mini_box[3][0] mini_box = mini_boxt mini_box = np.int32(mini_box) ret.append(mini_box) #scores.append(score_i) #print("---------------",mini_box) #cv2.polylines(im, [mini_box], 1, (255,255,255)) #submit_path = os.path.join(submit_dir,'res_img_{}.txt'.format(index)) #result_txt = open(submit_path,'a') #for i in range(0,4): # result_txt.write(str(mini_box[i][0])) # result_txt.write(',') # result_txt.write(str(mini_box[i][1])) # if i < 3: # result_txt.write(',') #result_txt.write('\r\n') #result_txt.close() return ret