def plot_n_set_results(evaluator, img_path, results_base_path, cfg): from matplotlib.pyplot import imsave # get image paths with open(cfg["DATA"].TEST_MAP_FILE) as f: content = f.readlines() img_base_path = os.path.dirname(os.path.abspath(cfg["DATA"].TEST_MAP_FILE)) img_file_names = [os.path.join(img_base_path, x.split('\t')[1]) for x in content] img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH) img_result = "{}/{}".format(results_base_path, os.path.basename(img_path)) out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed(img_path) labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, use_gpu_nms=cfg.USE_GPU_NMS, device_id=cfg.GPU_ID, nms_threshold=cfg.RESULTS_NMS_THRESHOLD, conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD) filtered_bboxes = regressed_rois[nmsKeepIndices] filtered_labels = labels[nmsKeepIndices] filtered_scores = scores[nmsKeepIndices] img, results = visualize_detections(img_path, filtered_bboxes, filtered_labels, filtered_scores, img_shape[2], img_shape[1], classes=cfg["DATA"].CLASSES, draw_negative_rois=cfg.DRAW_NEGATIVE_ROIS, decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD) imsave(img_result, img) return img_result, results
def plot_test_set_results(evaluator, num_images_to_plot, results_base_path, cfg): from matplotlib.pyplot import imsave from utils.rpn.bbox_transform import regress_rois from utils.nms_wrapper import apply_nms_to_single_image_results import json # get image paths with open(cfg["DATA"].TEST_MAP_FILE) as f: content = f.readlines() img_base_path = os.path.dirname( os.path.abspath(cfg["DATA"].TEST_MAP_FILE)) img_file_names = [ os.path.join(img_base_path, x.split('\t')[1]) for x in content ] img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH) print("Plotting results from for %s images." % num_images_to_plot) for i in range(0, num_images_to_plot): img_path = img_file_names[i] out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed( img_path) labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results( regressed_rois, labels, scores, use_gpu_nms=cfg.USE_GPU_NMS, device_id=cfg.GPU_ID, nms_threshold=cfg.RESULTS_NMS_THRESHOLD, conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD) filtered_bboxes = regressed_rois[nmsKeepIndices] filtered_labels = labels[nmsKeepIndices] filtered_scores = scores[nmsKeepIndices] json_output["images"][img_path] = { "class": cfg["DATA"].CLASSES[0], "bounding_boxes": [] } img = visualize_detections( img_path, filtered_bboxes, filtered_labels, filtered_scores, img_shape[2], img_shape[1], classes=cfg["DATA"].CLASSES, draw_negative_rois=cfg.DRAW_NEGATIVE_ROIS, decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD) imsave( "{}/{}_regr_{}".format(results_base_path, i, os.path.basename(img_path)), img)
def process_image(self, img_path): out_cls_pred, out_rpn_rois, out_bbox_regr, dims = self.process_image_detailed( img_path) labels = out_cls_pred.argmax(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) return regressed_rois, out_cls_pred
def plot_test_file_results(evaluator, path_to_file, results_base_path, cfg): from matplotlib.pyplot import imsave # get image paths img_path = path_to_file out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed( img_path) labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results( regressed_rois, labels, scores, use_gpu_nms=cfg.USE_GPU_NMS, device_id=cfg.GPU_ID, nms_threshold=cfg.RESULTS_NMS_THRESHOLD, conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD) filtered_bboxes = regressed_rois[nmsKeepIndices] filtered_labels = labels[nmsKeepIndices] filtered_scores = scores[nmsKeepIndices] img = visualize_detections( img_path, filtered_bboxes, filtered_labels, filtered_scores, img_shape[2], img_shape[1], classes=cfg["DATA"].CLASSES, draw_negative_rois=cfg.DRAW_NEGATIVE_ROIS, decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD) img_path = "{}/{}_regr_{}".format(results_base_path, 'result', os.path.basename(img_path)) imsave(img_path, img) return img_path
def get_results(evaluator, payload, cfg): # from matplotlib.pyplot import imsave img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH) # img_result = "{}/{}".format(results_base_path, os.path.basename(img_path)) pimg = np.fromstring(base64.b64decode(payload), np.uint8) source = cv2.imdecode(pimg, cv2.IMREAD_COLOR) out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed( source) labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results( regressed_rois, labels, scores, use_gpu_nms=cfg.USE_GPU_NMS, device_id=cfg.GPU_ID, nms_threshold=cfg.RESULTS_NMS_THRESHOLD, conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD) filtered_bboxes = regressed_rois[nmsKeepIndices] filtered_labels = labels[nmsKeepIndices] filtered_scores = scores[nmsKeepIndices] return serialization_detections( filtered_bboxes, filtered_labels, filtered_scores, img_shape[2], img_shape[1], classes=cfg["DATA"].CLASSES, decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD)
def inference(self, batch, **scalars): ''' Fill this method to write your own inference python code, you can refer to the model instance that is created in the load_model method. Expected results format is described in the returns as below. :param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length of ExtractBands in the emd. If BatchInference is set to False in emd, B is constant 1. :param scalars: inference parameters, accessed by the parameter name, i.e. score_threshold=float(scalars['score_threshold']). If you want to have more inference parameters, add it to the list of the following getParameterInfo method. :return: bounding boxes, python list representing bounding boxes whose length is equal to B, each element is [N,4] numpy array representing [ymin, xmin, ymax, xmax] with respect to the upper left corner of the image tile. scores, python list representing the score of each bounding box whose length is equal to B, each element is [N,] numpy array classes, python list representing the class of each bounding box whose length is equal to B, each element is [N,] numpy array and its dype is np.uint8 ''' # Todo: fill in this method to inference your model and return bounding boxes, scores and classes nms_threshold = float(scalars['nms_threshold']) batch = batch.astype(np.float32) batch_size, bands, height, width = batch.shape dims = np.array([width, height, width, height, width, height], np.float32) output = self.model.eval({ self.model.arguments[0]: batch, self.model.arguments[1]: [dims] }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results( regressed_rois, labels, scores, False, 0, nms_threshold=nms_threshold, conf_threshold=self.score_threshold) filtered_bboxes = regressed_rois[nmsKeepIndices] filtered_scores = scores[nmsKeepIndices] filtered_labels = labels[nmsKeepIndices] positive_label_indices = np.where(filtered_labels > 0) filtered_bboxes = filtered_bboxes[positive_label_indices] filtered_scores = filtered_scores[positive_label_indices] filtered_labels = filtered_labels[positive_label_indices] filtered_bboxes = filtered_bboxes[:, [1, 0, 3, 2]] return [filtered_bboxes], [filtered_scores], [filtered_labels]
def compute_test_set_aps(eval_model, cfg): num_test_images = cfg["DATA"].NUM_TEST_IMAGES classes = cfg["DATA"].CLASSES image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH), dynamic_axes=[Axis.default_batch_axis()], name=cfg["MODEL"].FEATURE_NODE_NAME) roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()]) dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()]) frcn_eval = eval_model(image_input, dims_input) # Create the minibatch source minibatch_source = ObjectDetectionMinibatchSource( cfg["DATA"].TEST_MAP_FILE, cfg["DATA"].TEST_ROI_FILE, max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE, pad_width=cfg.IMAGE_WIDTH, pad_height=cfg.IMAGE_HEIGHT, pad_value=cfg["MODEL"].IMG_PAD_COLOR, randomize=False, use_flipping=False, max_images=cfg["DATA"].NUM_TEST_IMAGES, num_classes=cfg["DATA"].NUM_CLASSES, proposal_provider=None) # define mapping from reader streams to network inputs input_map = { minibatch_source.image_si: image_input, minibatch_source.roi_si: roi_input, minibatch_source.dims_si: dims_input } # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(num_test_images)] for _ in range(cfg["DATA"].NUM_CLASSES)] # evaluate test images and write netwrok output to file print("Evaluating Faster R-CNN model for %s images." % num_test_images) all_gt_infos = {key: [] for key in classes} for img_i in range(0, num_test_images): mb_data = minibatch_source.next_minibatch(1, input_map=input_map) gt_row = mb_data[roi_input].asarray() gt_row = gt_row.reshape((cfg.INPUT_ROIS_PER_IMAGE, 5)) all_gt_boxes = gt_row[np.where(gt_row[:, -1] > 0)] for cls_index, cls_name in enumerate(classes): if cls_index == 0: continue cls_gt_boxes = all_gt_boxes[np.where( all_gt_boxes[:, -1] == cls_index)] all_gt_infos[cls_name].append({ 'bbox': np.array(cls_gt_boxes), 'difficult': [False] * len(cls_gt_boxes), 'det': [False] * len(cls_gt_boxes) }) output = frcn_eval.eval({ image_input: mb_data[image_input], dims_input: mb_data[dims_input] }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, mb_data[dims_input].asarray()) labels.shape = labels.shape + (1, ) scores.shape = scores.shape + (1, ) coords_score_label = np.hstack((regressed_rois, scores, labels)) # shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score for cls_j in range(1, cfg["DATA"].NUM_CLASSES): coords_score_label_for_cls = coords_score_label[np.where( coords_score_label[:, -1] == cls_j)] all_boxes[cls_j][ img_i] = coords_score_label_for_cls[:, :-1].astype(np.float32, copy=False) if (img_i + 1) % 100 == 0: print("Processed {} samples".format(img_i + 1)) # calculate mAP aps = evaluate_detections(all_boxes, all_gt_infos, classes, use_gpu_nms=cfg.USE_GPU_NMS, device_id=cfg.GPU_ID, nms_threshold=cfg.RESULTS_NMS_THRESHOLD, conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD) return aps
def compute_test_set_aps(eval_model, cfg): num_test_images = cfg["DATA"].NUM_TEST_IMAGES classes = cfg["DATA"].CLASSES image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH), dynamic_axes=[Axis.default_batch_axis()], name=cfg["MODEL"].FEATURE_NODE_NAME) roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()]) roi_proposals = input_variable((cfg.NUM_ROI_PROPOSALS, 4), dynamic_axes=[Axis.default_batch_axis()], name="roi_proposals") dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()]) frcn_eval = eval_model(image_input, roi_proposals) # Create the minibatch source if cfg.USE_PRECOMPUTED_PROPOSALS: try: cfg["DATA"].TEST_PRECOMPUTED_PROPOSALS_FILE = os.path.join(cfg["DATA"].MAP_FILE_PATH, cfg["DATA"].TEST_PRECOMPUTED_PROPOSALS_FILE) proposal_provider = ProposalProvider.fromfile(cfg["DATA"].TEST_PRECOMPUTED_PROPOSALS_FILE, cfg.NUM_ROI_PROPOSALS) except: print("To use precomputed proposals please specify the following parameters in your configuration:\n" "__C.DATA.TRAIN_PRECOMPUTED_PROPOSALS_FILE\n" "__C.DATA.TEST_PRECOMPUTED_PROPOSALS_FILE") exit(-1) else: proposal_provider = ProposalProvider.fromconfig(cfg) minibatch_source = ObjectDetectionMinibatchSource( cfg["DATA"].TEST_MAP_FILE, cfg["DATA"].TEST_ROI_FILE, max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE, pad_width=cfg.IMAGE_WIDTH, pad_height=cfg.IMAGE_HEIGHT, pad_value=cfg["MODEL"].IMG_PAD_COLOR, randomize=False, use_flipping=False, max_images=cfg["DATA"].NUM_TEST_IMAGES, num_classes=cfg["DATA"].NUM_CLASSES, proposal_provider=proposal_provider, provide_targets=False) # define mapping from reader streams to network inputs input_map = { minibatch_source.image_si: image_input, minibatch_source.roi_si: roi_input, minibatch_source.proposals_si: roi_proposals, minibatch_source.dims_si: dims_input } # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(num_test_images)] for _ in range(cfg["DATA"].NUM_CLASSES)] # evaluate test images and write netwrok output to file print("Evaluating Fast R-CNN model for %s images." % num_test_images) all_gt_infos = {key: [] for key in classes} for img_i in range(0, num_test_images): mb_data = minibatch_source.next_minibatch(1, input_map=input_map) gt_row = mb_data[roi_input].asarray() gt_row = gt_row.reshape((cfg.INPUT_ROIS_PER_IMAGE, 5)) all_gt_boxes = gt_row[np.where(gt_row[:,-1] > 0)] for cls_index, cls_name in enumerate(classes): if cls_index == 0: continue cls_gt_boxes = all_gt_boxes[np.where(all_gt_boxes[:,-1] == cls_index)] all_gt_infos[cls_name].append({'bbox': np.array(cls_gt_boxes), 'difficult': [False] * len(cls_gt_boxes), 'det': [False] * len(cls_gt_boxes)}) output = frcn_eval.eval({image_input: mb_data[image_input], roi_proposals: mb_data[roi_proposals]}) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = mb_data[roi_proposals].data.asarray() out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, mb_data[dims_input].asarray()) labels.shape = labels.shape + (1,) scores.shape = scores.shape + (1,) coords_score_label = np.hstack((regressed_rois, scores, labels)) # shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score for cls_j in range(1, cfg["DATA"].NUM_CLASSES): coords_score_label_for_cls = coords_score_label[np.where(coords_score_label[:,-1] == cls_j)] all_boxes[cls_j][img_i] = coords_score_label_for_cls[:,:-1].astype(np.float32, copy=False) if (img_i+1) % 100 == 0: print("Processed {} samples".format(img_i+1)) # calculate mAP aps = evaluate_detections(all_boxes, all_gt_infos, classes, use_gpu_nms = cfg.USE_GPU_NMS, device_id = cfg.GPU_ID, nms_threshold=cfg.RESULTS_NMS_THRESHOLD, conf_threshold = cfg.RESULTS_NMS_CONF_THRESHOLD) return aps
def process_image(self, img_path): out_cls_pred, out_rpn_rois, out_bbox_regr, dims = self.process_image_detailed(img_path) labels = out_cls_pred.argmax(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) return regressed_rois, out_cls_pred