def eval_and_plot_faster_rcnn(eval_model, num_images_to_plot, test_map_file, img_shape, results_base_path, feature_node_name, classes, drawUnregressedRois=False, drawNegativeRois=False, nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8): # get image paths with open(test_map_file) as f: content = f.readlines() img_base_path = os.path.dirname(os.path.abspath(test_map_file)) img_file_names = [os.path.join(img_base_path, x.split('\t')[1]) for x in content] # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) #dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6)) print("Plotting results from Faster R-CNN model for %s images." % num_images_to_plot) for i in range(0, num_images_to_plot): imgPath = img_file_names[i] # evaluate single image _, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1,) + dims_input.shape output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input}) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() if drawUnregressedRois: # plot results without final regression imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave("{}/{}_{}".format(results_base_path, i, os.path.basename(imgPath)), imgDebug) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, nms_threshold=nmsThreshold, conf_threshold=nmsConfThreshold) img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=nmsKeepIndices, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave("{}/{}_regr_{}".format(results_base_path, i, os.path.basename(imgPath)), img)
def eval_and_plot_faster_rcnn(eval_model, num_images_to_plot, test_map_file, img_shape, results_base_path, feature_node_name, classes, drawUnregressedRois=False, drawNegativeRois=False, nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8): # get image paths with open(test_map_file) as f: content = f.readlines() img_base_path = os.path.dirname(os.path.abspath(test_map_file)) img_file_names = [os.path.join(img_base_path, x.split('\t')[1]) for x in content] # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) #dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6)) print("Plotting results from Faster R-CNN model for %s images." % num_images_to_plot) for i in range(0, num_images_to_plot): imgPath = img_file_names[i] # evaluate single image _, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1,) + dims_input.shape output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input}) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() if drawUnregressedRois: # plot results without final regression imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave("{}/{}_{}".format(results_base_path, i, os.path.basename(imgPath)), imgDebug) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, nms_threshold=nmsThreshold, conf_threshold=nmsConfThreshold) img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=nmsKeepIndices, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave("{}/{}_regr_{}".format(results_base_path, i, os.path.basename(imgPath)), img)
def eval_faster_rcnn_mAP(eval_model): img_map_file = globalvars['test_map_file'] roi_map_file = globalvars['test_roi_file'] classes = globalvars['classes'] image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()]) dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()]) frcn_eval = eval_model(image_input, dims_input) # Create the minibatch source minibatch_source = ObjectDetectionMinibatchSource( img_map_file, roi_map_file, max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE, pad_width=image_width, pad_height=image_height, pad_value=img_pad_value, randomize=False, use_flipping=False, max_images=cfg["CNTK"].NUM_TEST_IMAGES) # define mapping from reader streams to network inputs input_map = { minibatch_source.image_si: image_input, minibatch_source.roi_si: roi_input, minibatch_source.dims_si: dims_input } # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in # (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(num_test_images)] for _ in range(globalvars['num_classes'])] # evaluate test images and write netwrok output to file print("Evaluating Faster R-CNN model for %s images." % num_test_images) all_gt_infos = {key: [] for key in classes} for img_i in range(0, num_test_images): mb_data = minibatch_source.next_minibatch(1, input_map=input_map) gt_row = mb_data[roi_input].asarray() gt_row = gt_row.reshape((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5)) all_gt_boxes = gt_row[np.where(gt_row[:, -1] > 0)] for cls_index, cls_name in enumerate(classes): if cls_index == 0: continue cls_gt_boxes = all_gt_boxes[np.where( all_gt_boxes[:, -1] == cls_index)] all_gt_infos[cls_name].append({ 'bbox': np.array(cls_gt_boxes), 'difficult': [False] * len(cls_gt_boxes), 'det': [False] * len(cls_gt_boxes) }) output = frcn_eval.eval({ image_input: mb_data[image_input], dims_input: mb_data[dims_input] }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, mb_data[dims_input].asarray()) labels.shape = labels.shape + (1, ) scores.shape = scores.shape + (1, ) coords_score_label = np.hstack((regressed_rois, scores, labels)) # shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score for cls_j in range(1, globalvars['num_classes']): coords_score_label_for_cls = coords_score_label[np.where( coords_score_label[:, -1] == cls_j)] all_boxes[cls_j][ img_i] = coords_score_label_for_cls[:, :-1].astype(np.float32, copy=False) if (img_i + 1) % 100 == 0: print("Processed {} samples".format(img_i + 1)) confusions = None try: conf_file = cfg["CNTK"].CONFUSION_FILE conf_file = os.path.join(map_file_path, conf_file) confusions = confusions_map(classes, conf_file) except: confusions = None # calculate mAP aps, fp_errors = evaluate_detections( all_boxes, all_gt_infos, classes, nms_threshold=cfg["CNTK"].RESULTS_NMS_THRESHOLD, conf_threshold=cfg["CNTK"].RESULTS_NMS_CONF_THRESHOLD, soft=cfg["CNTK"].RESULTS_NMS_SOFT, confusions=confusions) if fp_errors: output_file = os.path.join( globalvars['output_path'], "{}_{}_fps.txt".format( cfg["CNTK"].BASE_MODEL, "e2e" if globalvars['train_e2e'] else "4stage")) log_fp_errors(fp_errors, output_file) ap_list = [] for class_name in aps: ap_list += [aps[class_name]] print('AP for {:>15} = {:.4f}'.format(class_name, aps[class_name])) meanAP = np.nanmean(ap_list) print('Mean AP = {:.4f}'.format(meanAP)) return meanAP
def faster_rcnn_pred(eval_model, prediction_in, prediction_out, class_map_file, img_shape, feature_node_name, nms_threshold=0.5, nms_conf_threshold=0.0, headers=True, output_width_height=False, suppressed_labels=()): """ Performs predictions with a model in continuous mode. :param eval_model: the model to use :param prediction_in: the input dir for images (list) :param prediction_out: the output dir for images and ROI CSV files (list) :param class_map_file: the class map file to use :param img_shape: the shpage :param feature_node_name: the name of the feature node :param nms_threshold: :param nms_conf_threshold: :param headers: whether to output the headers in the ROI file :param output_width_height: whether to output x/y/width/height or x0/y0/x1/y1 :param suppressed_labels: the labels to suppress from being output in the ROI files """ # load labels str_labels = load_class_labels(class_map_file) print("Class labels: %s" % str(str_labels)) # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1, 6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) while True: any = False # iterate directory pairs for p in range(len(prediction_in)): pred_in = prediction_in[p] pred_out = prediction_out[p] # only png and jpg files files = [ (pred_in + os.sep + x) for x in os.listdir(pred_in) if (x.lower().endswith(".png") or x.lower().endswith(".jpg")) ] for f in files: start = datetime.now() print(start, "-", f) img_path = pred_out + os.sep + os.path.basename(f) roi_path = pred_out + os.sep + os.path.splitext( os.path.basename(f))[0] + ".csv" cntk_img_input = None try: _, cntk_img_input, dims = load_resize_and_pad( f, img_shape[2], img_shape[1]) except Exception as e: print(str(e)) try: # delete any existing old files in output dir if os.path.exists(img_path): try: os.remove(img_path) except: print( "Failed to remove existing image in output directory: ", img_path) if os.path.exists(roi_path): try: os.remove(roi_path) except: print( "Failed to remove existing ROI file in output directory: ", roi_path) # move into output dir os.rename(f, img_path) except: img_path = None if cntk_img_input is None: continue if img_path is None: continue dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1, ) + dims_input.shape output = frcn_eval.eval({ frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nms_keep_indices = apply_nms_to_single_image_results( regressed_rois, labels, scores, nms_threshold=nms_threshold, conf_threshold=nms_conf_threshold) save_rois_to_file(regressed_rois, nms_keep_indices, labels, str_labels, scores, pred_out, img_path, headers=headers, output_width_height=output_width_height, suppressed_labels=suppressed_labels, dims=dims) timediff = datetime.now() - start print(" time:", timediff) # nothing processed at all, lets wait for files to appear if not any: sleep(1)
def faster_rcnn_eval(eval_model, test_map_file, class_map_file, img_shape, results_base_path, feature_node_name, nms_threshold=0.5, nms_conf_threshold=0.0, headers=True, output_width_height=False, suppressed_labels=()): """ Tests a Faster R-CNN model and outputs rois. :param eval_model: the model :param test_map_file: the map file with the test images/labels :param class_map_file: the class map file :param img_shape: the shape :param results_base_path: the base directory for the results :param feature_node_name: the feature node name :param nms_threshold: :param nms_conf_threshold: :param headers: whether to output the headers in the ROI file :param output_width_height: whether to output x/y/width/height or x0/y0/x1/y1 :param suppressed_labels: the labels to suppress from being output in the ROI files """ # load labels str_labels = load_class_labels(class_map_file) print("Class labels: %s" % str(str_labels)) # get image paths with open(test_map_file) as f: content = f.readlines() img_base_path = os.path.dirname(os.path.abspath(test_map_file)) img_file_names = [ os.path.join(img_base_path, x.split('\t')[1]) for x in content ] # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1, 6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) print("Outputting results from Faster R-CNN model for %s images." % len(img_file_names)) for i in range(0, len(img_file_names)): start = datetime.now() img_path = img_file_names[i] print( str(i + 1) + "/" + str(len(img_file_names)) + ": " + img_file_names[i]) # evaluate single image _, cntk_img_input, dims = load_resize_and_pad(img_path, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1, ) + dims_input.shape output = frcn_eval.eval({ frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nms_keep_indices = apply_nms_to_single_image_results( regressed_rois, labels, scores, nms_threshold=nms_threshold, conf_threshold=nms_conf_threshold) save_rois_to_file(regressed_rois, nms_keep_indices, labels, str_labels, scores, results_base_path, img_path, headers=headers, output_width_height=output_width_height, suppressed_labels=suppressed_labels, dims=dims) timediff = datetime.now() - start print(" time:", timediff)
def faster_rcnn_eval_and_plot(eval_model, num_images_to_plot, test_map_file, class_map_file, img_shape, results_base_path, feature_node_name, classes, draw_unregressed_rois=False, draw_negative_rois=False, nms_threshold=0.5, nms_conf_threshold=0.0, bgr_plot_threshold=0.8, headers=True, output_width_height=False, suppressed_labels=()): """ Tests a Faster R-CNN model and plots images with detected boxes. :param eval_model: the model to evaluate :param num_images_to_plot: the number of images to plot :param test_map_file: the map file with the test images/labels :param class_map_file: the class map file :param img_shape: the shape :param results_base_path: the base of the output directory :param feature_node_name: the feature node name :param classes: :param draw_unregressed_rois: whether to draw unregressed ROIs :param draw_negative_rois: whether to draw negative ROIs (eg background) :param nms_threshold: :param nms_conf_threshold: :param bgr_plot_threshold: :param headers: whether to output the headers in the ROI file :param output_width_height: whether to output x/y/width/height or x0/y0/x1/y1 """ # load labels str_labels = load_class_labels(class_map_file) print("Class labels: %s" % str(str_labels)) # get image paths with open(test_map_file) as f: content = f.readlines() img_base_path = os.path.dirname(os.path.abspath(test_map_file)) img_file_names = [ os.path.join(img_base_path, x.split('\t')[1]) for x in content ] # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1, 6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) # dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6)) print("Plotting results from Faster R-CNN model for %s images." % num_images_to_plot) for i in range(0, num_images_to_plot): img_path = img_file_names[i] # save padded image write_padded_image( img_path, "{}/{}-padded.jpg".format( results_base_path, os.path.splitext(os.path.basename(img_path))[0])) # evaluate single image _, cntk_img_input, dims = load_resize_and_pad(img_path, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1, ) + dims_input.shape output = frcn_eval.eval({ frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() if draw_unregressed_rois: # plot results without final regression img_debug = faster_rcnn_visualize_results( img_path, labels, scores, out_rpn_rois, img_shape[2], img_shape[1], classes, nms_keep_indices=None, draw_negative_rois=draw_negative_rois, decision_threshold=bgr_plot_threshold) imsave( "{}/{}".format(results_base_path, os.path.basename(img_path)), img_debug) # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nms_keep_indices = apply_nms_to_single_image_results( regressed_rois, labels, scores, nms_threshold=nms_threshold, conf_threshold=nms_conf_threshold) save_rois_to_file(regressed_rois, nms_keep_indices, labels, str_labels, scores, results_base_path, img_path, headers=headers, output_width_height=output_width_height, suppressed_labels=suppressed_labels, dims=dims) img = faster_rcnn_visualize_results( img_path, labels, scores, regressed_rois, img_shape[2], img_shape[1], classes, nms_keep_indices=nms_keep_indices, draw_negative_rois=draw_negative_rois, decision_threshold=bgr_plot_threshold) imsave( "{}/{}-regr.jpg".format( results_base_path, os.path.splitext(os.path.basename(img_path)))[0], img)
def eval_faster_rcnn(eval_model, imgPath, img_shape, results_base_path, feature_node_name, classes, mode, drawUnregressedRois=False, drawNegativeRois=False, nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8): # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) #dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6)) print("Plotting results from Faster R-CNN model for image.") # evaluate single image _, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1,) + dims_input.shape output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input}) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() if mode=="returntags": class Tag(object): def __init__(self, label, score, bbox): self.label = label self.score = score self.bbox = bbox def serialize(self): return { 'label': self.label, 'score': self.score, 'bbox': self.bbox, } results = [] for i in range(len(out_rpn_rois)): if labels[i] != 0: x = Tag(str(classes[labels[i]]), str(scores[i]), str(out_rpn_rois[i])) results.append(x) return results elif mode=="returnimage": evaluated_image_path = "{}/{}".format(results_base_path, 'evaluated_' + os.path.basename(imgPath)) if drawUnregressedRois: # plot results without final regression imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave(evaluated_image_path, imgDebug) else: # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, nms_threshold=nmsThreshold, conf_threshold=nmsConfThreshold) img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=nmsKeepIndices, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave(evaluated_image_path, img) return evaluated_image_path else: raise ValueError("Unsupported value found in 'mode' parameter")
def eval_faster_rcnn(eval_model, imgPath, img_shape, results_base_path, feature_node_name, classes, mode, drawUnregressedRois=False, drawNegativeRois=False, nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold=0.8): # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1, 6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') # if your model hasn't been loaded proper or in the proper place this is the line to look at. This line takes a model and loads it # in preparation for the evaluation try: frcn_eval = eval_model(image_input, dims_input) except: raise TypeError("Loading existing model from %s" % model_path) print("Plotting results from Faster R-CNN model for image.") _, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1, ) + dims_input.shape output = frcn_eval.eval({ frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input }) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() if mode == "returntags": class Tag(object): def __init__(self, label, score, bbox): self.label = label self.score = score self.bbox = bbox def serialize(self): return { 'label': self.label, 'score': self.score, 'bbox': self.bbox, } results = [] for i in range(len(out_rpn_rois)): if labels[i] != 0: x = Tag(str(classes[labels[i]]), str(scores[i]), str(out_rpn_rois[i])) results.append(x) return results elif mode == "returnimage": evaluated_image_path = "{}/{}".format( results_base_path, 'evaluated_' + os.path.basename(imgPath)) if drawUnregressedRois: # plot results without final regression imgDebug = visualizeResultsFaster( imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave(evaluated_image_path, imgDebug) else: # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results( regressed_rois, labels, scores, nms_threshold=nmsThreshold, conf_threshold=nmsConfThreshold) img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=nmsKeepIndices, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave(evaluated_image_path, img) return evaluated_image_path else: raise ValueError("Unsupported value found in 'mode' parameter")
def eval_faster_rcnn_mAP(eval_model): img_map_file = globalvars['test_map_file'] roi_map_file = globalvars['test_roi_file'] classes = globalvars['classes'] image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()]) dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()]) frcn_eval = eval_model(image_input, dims_input) # Create the minibatch source minibatch_source = ObjectDetectionMinibatchSource( img_map_file, roi_map_file, max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE, pad_width=image_width, pad_height=image_height, pad_value=img_pad_value, randomize=False, use_flipping=False, max_images=cfg["CNTK"].NUM_TEST_IMAGES) # define mapping from reader streams to network inputs input_map = { minibatch_source.image_si: image_input, minibatch_source.roi_si: roi_input, minibatch_source.dims_si: dims_input } # all detections are collected into: # all_boxes[cls][image] = N x 5 array of detections in # (x1, y1, x2, y2, score) all_boxes = [[[] for _ in range(num_test_images)] for _ in range(globalvars['num_classes'])] # evaluate test images and write netwrok output to file print("Evaluating Faster R-CNN model for %s images." % num_test_images) all_gt_infos = {key: [] for key in classes} for img_i in range(0, num_test_images): mb_data = minibatch_source.next_minibatch(1, input_map=input_map) gt_row = mb_data[roi_input].asarray() gt_row = gt_row.reshape((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5)) all_gt_boxes = gt_row[np.where(gt_row[:,-1] > 0)] for cls_index, cls_name in enumerate(classes): if cls_index == 0: continue cls_gt_boxes = all_gt_boxes[np.where(all_gt_boxes[:,-1] == cls_index)] all_gt_infos[cls_name].append({'bbox': np.array(cls_gt_boxes), 'difficult': [False] * len(cls_gt_boxes), 'det': [False] * len(cls_gt_boxes)}) output = frcn_eval.eval({image_input: mb_data[image_input], dims_input: mb_data[dims_input]}) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1) regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, mb_data[dims_input].asarray()) labels.shape = labels.shape + (1,) scores.shape = scores.shape + (1,) coords_score_label = np.hstack((regressed_rois, scores, labels)) # shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score for cls_j in range(1, globalvars['num_classes']): coords_score_label_for_cls = coords_score_label[np.where(coords_score_label[:,-1] == cls_j)] all_boxes[cls_j][img_i] = coords_score_label_for_cls[:,:-1].astype(np.float32, copy=False) if (img_i+1) % 100 == 0: print("Processed {} samples".format(img_i+1)) # calculate mAP aps = evaluate_detections(all_boxes, all_gt_infos, classes, nms_threshold=cfg["CNTK"].RESULTS_NMS_THRESHOLD, conf_threshold = cfg["CNTK"].RESULTS_NMS_CONF_THRESHOLD) ap_list = [] for class_name in aps: ap_list += [aps[class_name]] print('AP for {:>15} = {:.4f}'.format(class_name, aps[class_name])) meanAP = np.nanmean(ap_list) print('Mean AP = {:.4f}'.format(meanAP)) return meanAP
def eval_faster_rcnn(eval_model, imgPath, img_shape, results_base_path, feature_node_name, classes, mode, drawUnregressedRois=False, drawNegativeRois=False, nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8): # prepare model image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name) dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input') frcn_eval = eval_model(image_input, dims_input) #dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6)) print("Plotting results from Faster R-CNN model for image.") # evaluate single image _, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1]) dims_input = np.array(dims, dtype=np.float32) dims_input.shape = (1,) + dims_input.shape output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input}) out_dict = dict([(k.name, k) for k in output]) out_cls_pred = output[out_dict['cls_pred']][0] out_rpn_rois = output[out_dict['rpn_rois']][0] out_bbox_regr = output[out_dict['bbox_regr']][0] labels = out_cls_pred.argmax(axis=1) scores = out_cls_pred.max(axis=1).tolist() if mode=="returntags": class Tag(object): def __init__(self, label, score, bbox): self.label = label self.score = score self.bbox = bbox def serialize(self): return { 'label': self.label, 'score': self.score, 'bbox': self.bbox, } results = [] regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, nms_threshold=nmsThreshold, conf_threshold=nmsConfThreshold) print(len(out_rpn_rois)) imsave('./Temp/resized.jpg',imgDebug) for i in range(len(out_rpn_rois)): if labels[i] != 0: x = Tag(str(classes[labels[i]]), str(scores[i]), str(out_rpn_rois[i])) results.append(x) # return {} return results elif mode=="returnimage": evaluated_image_path = "{}/{}".format(results_base_path, 'evaluated_' + os.path.basename(imgPath)) if drawUnregressedRois: # plot results without final regression imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) imsave(evaluated_image_path, imgDebug) else: # apply regression and nms to bbox coordinates regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims) nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores, nms_threshold=nmsThreshold, conf_threshold=nmsConfThreshold) img,allboxes = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1], classes, nmsKeepIndices=nmsKeepIndices, boDrawNegativeRois=drawNegativeRois, decisionThreshold=bgrPlotThreshold) # imsave(evaluated_image_path, img) allboxes=np.array(allboxes) # perform non-maximum suppression on the bounding boxes pick = non_max_suppression_fast(allboxes, 0.6) # print("[x] after applying non-maximum, %d bounding boxes" % (len(pick))) black_bg = 0*np.ones_like(img) # loop over the picked bounding boxes and extract each of the box for (startX, startY, endX, endY) in pick: roi=img[startY:endY,startX:endX] black_bg[startY:endY,startX:endX]=roi result = black_bg.copy() print(black_bg.shape) image = cv2.cvtColor(black_bg, cv2.COLOR_RGB2HSV) lower = np.array([18, 0, 0]) upper = np.array([179, 255, 255]) mask = cv2.inRange(image, lower, upper) result = cv2.bitwise_and(result,result, mask=mask) lengthThroughRotatedRectangle=[] lengthThroughManualCalculation=[] # length Calculation through Rotated Rectangle image=cv2.cvtColor(result,cv2.COLOR_BGR2GRAY) ret,thresh = cv2.threshold(image,127,255,0) working_image=thresh.copy() result_img=thresh.copy() working_image[:,:]=0 result_img[:,:]=0 kernel = np.ones((7,7), np.uint8) for (startX, startY, endX, endY) in pick: cord=(int(startX),int(startY),int(endX),int(endY)) working_image[:,:]=0 working_image[cord[1]:cord[3],cord[0]:cord[2]]=thresh[cord[1]:cord[3],cord[0]:cord[2]] result_img[cord[1]:cord[3],cord[0]:cord[2]]=thresh[cord[1]:cord[3],cord[0]:cord[2]] working_image=cv2.morphologyEx(working_image, cv2.MORPH_OPEN, kernel) contours, hierarchy = cv2.findContours(working_image,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) if(len(contours)>0): c = max(contours, key = cv2.contourArea) rect = cv2.minAreaRect(c) (x, y), (width, height), angle=rect height=int(height) width=int(width) if width >height: height=width lengthThroughRotatedRectangle.append(height); # length Calculation through Manual process for (startX, startY, endX, endY) in pick: cord=(int(startX),int(startY),int(endX),int(endY)) widthofRectangle=cord[2]-cord[0] NonZeroPixels=[] threshold=0.20; height=0 width=0 for i in range(startY,endY): row=thresh[i,startX:endX] NonZeroPixelsInRow=np.count_nonzero(row) WidthRatioInRow=NonZeroPixelsInRow/widthofRectangle if(WidthRatioInRow>threshold): height=height+1 NonZeroPixels.append(NonZeroPixelsInRow) width=round(sum(NonZeroPixels)/len(NonZeroPixels)) if(width > height): height=width lengthThroughManualCalculation.append(height) # print("length through rotatedRectangle\n",lengthThroughRotatedRectangle) # print("length through ManualCalculation\n",lengthThroughManualCalculation) SpikesLength=[int((a+b)/2) for a,b in zip(lengthThroughRotatedRectangle,lengthThroughManualCalculation)] # print("spike length\n",SpikesLength) SpikesLength = [i * 0.1 for i in SpikesLength] print("Spike Lenght in cm",SpikesLength) # imsave(evaluated_image_path, thresh) return SpikesLength else: raise ValueError("Unsupported value found in 'mode' parameter")