def plot_n_set_results(evaluator, img_path, results_base_path, cfg):
from matplotlib.pyplot import imsave
# get image paths
with open(cfg["DATA"].TEST_MAP_FILE) as f:
content = f.readlines()
img_base_path = os.path.dirname(os.path.abspath(cfg["DATA"].TEST_MAP_FILE))
img_file_names = [os.path.join(img_base_path, x.split('\t')[1]) for x in content]
img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH)
img_result = "{}/{}".format(results_base_path, os.path.basename(img_path))
out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed(img_path)
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores,
use_gpu_nms=cfg.USE_GPU_NMS,
device_id=cfg.GPU_ID,
nms_threshold=cfg.RESULTS_NMS_THRESHOLD,
conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD)
filtered_bboxes = regressed_rois[nmsKeepIndices]
filtered_labels = labels[nmsKeepIndices]
filtered_scores = scores[nmsKeepIndices]
img, results = visualize_detections(img_path, filtered_bboxes, filtered_labels, filtered_scores,
img_shape[2], img_shape[1],
classes=cfg["DATA"].CLASSES,
draw_negative_rois=cfg.DRAW_NEGATIVE_ROIS,
decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD)
imsave(img_result, img)
return img_result, results
def plot_test_set_results(evaluator, num_images_to_plot, results_base_path,
cfg):
from matplotlib.pyplot import imsave
from utils.rpn.bbox_transform import regress_rois
from utils.nms_wrapper import apply_nms_to_single_image_results
import json
# get image paths
with open(cfg["DATA"].TEST_MAP_FILE) as f:
content = f.readlines()
img_base_path = os.path.dirname(
os.path.abspath(cfg["DATA"].TEST_MAP_FILE))
img_file_names = [
os.path.join(img_base_path,
x.split('\t')[1]) for x in content
]
img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH)
print("Plotting results from for %s images." % num_images_to_plot)
for i in range(0, num_images_to_plot):
img_path = img_file_names[i]
out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed(
img_path)
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
dims)
nmsKeepIndices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
use_gpu_nms=cfg.USE_GPU_NMS,
device_id=cfg.GPU_ID,
nms_threshold=cfg.RESULTS_NMS_THRESHOLD,
conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD)
filtered_bboxes = regressed_rois[nmsKeepIndices]
filtered_labels = labels[nmsKeepIndices]
filtered_scores = scores[nmsKeepIndices]
json_output["images"][img_path] = {
"class": cfg["DATA"].CLASSES[0],
"bounding_boxes": []
}
img = visualize_detections(
img_path,
filtered_bboxes,
filtered_labels,
filtered_scores,
img_shape[2],
img_shape[1],
classes=cfg["DATA"].CLASSES,
draw_negative_rois=cfg.DRAW_NEGATIVE_ROIS,
decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD)
imsave(
"{}/{}_regr_{}".format(results_base_path, i,
os.path.basename(img_path)), img)
def process_image(self, img_path):
out_cls_pred, out_rpn_rois, out_bbox_regr, dims = self.process_image_detailed(
img_path)
labels = out_cls_pred.argmax(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
dims)
return regressed_rois, out_cls_pred
def plot_test_file_results(evaluator, path_to_file, results_base_path, cfg):
from matplotlib.pyplot import imsave
# get image paths
img_path = path_to_file
out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed(
img_path)
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
use_gpu_nms=cfg.USE_GPU_NMS,
device_id=cfg.GPU_ID,
nms_threshold=cfg.RESULTS_NMS_THRESHOLD,
conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD)
filtered_bboxes = regressed_rois[nmsKeepIndices]
filtered_labels = labels[nmsKeepIndices]
filtered_scores = scores[nmsKeepIndices]
img = visualize_detections(
img_path,
filtered_bboxes,
filtered_labels,
filtered_scores,
img_shape[2],
img_shape[1],
classes=cfg["DATA"].CLASSES,
draw_negative_rois=cfg.DRAW_NEGATIVE_ROIS,
decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD)
img_path = "{}/{}_regr_{}".format(results_base_path, 'result',
os.path.basename(img_path))
imsave(img_path, img)
return img_path
def get_results(evaluator, payload, cfg):
# from matplotlib.pyplot import imsave
img_shape = (cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH)
# img_result = "{}/{}".format(results_base_path, os.path.basename(img_path))
pimg = np.fromstring(base64.b64decode(payload), np.uint8)
source = cv2.imdecode(pimg, cv2.IMREAD_COLOR)
out_cls_pred, out_rpn_rois, out_bbox_regr, dims = evaluator.process_image_detailed(
source)
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
use_gpu_nms=cfg.USE_GPU_NMS,
device_id=cfg.GPU_ID,
nms_threshold=cfg.RESULTS_NMS_THRESHOLD,
conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD)
filtered_bboxes = regressed_rois[nmsKeepIndices]
filtered_labels = labels[nmsKeepIndices]
filtered_scores = scores[nmsKeepIndices]
return serialization_detections(
filtered_bboxes,
filtered_labels,
filtered_scores,
img_shape[2],
img_shape[1],
classes=cfg["DATA"].CLASSES,
decision_threshold=cfg.RESULTS_BGR_PLOT_THRESHOLD)
def inference(self, batch, **scalars):
'''
Fill this method to write your own inference python code, you can refer to the model instance that is created
in the load_model method. Expected results format is described in the returns as below.
:param batch: numpy array with shape (B, H, W, D), B is batch size, H, W is specified and equal to
ImageHeight and ImageWidth in the emd file and D is the number of bands and equal to the length
of ExtractBands in the emd. If BatchInference is set to False in emd, B is constant 1.
:param scalars: inference parameters, accessed by the parameter name,
i.e. score_threshold=float(scalars['score_threshold']). If you want to have more inference
parameters, add it to the list of the following getParameterInfo method.
:return: bounding boxes, python list representing bounding boxes whose length is equal to B, each element is
[N,4] numpy array representing [ymin, xmin, ymax, xmax] with respect to the upper left
corner of the image tile.
scores, python list representing the score of each bounding box whose length is equal to B, each element
is [N,] numpy array
classes, python list representing the class of each bounding box whose length is equal to B, each element
is [N,] numpy array and its dype is np.uint8
'''
# Todo: fill in this method to inference your model and return bounding boxes, scores and classes
nms_threshold = float(scalars['nms_threshold'])
batch = batch.astype(np.float32)
batch_size, bands, height, width = batch.shape
dims = np.array([width, height, width, height, width, height],
np.float32)
output = self.model.eval({
self.model.arguments[0]: batch,
self.model.arguments[1]: [dims]
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
dims)
nmsKeepIndices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
False,
0,
nms_threshold=nms_threshold,
conf_threshold=self.score_threshold)
filtered_bboxes = regressed_rois[nmsKeepIndices]
filtered_scores = scores[nmsKeepIndices]
filtered_labels = labels[nmsKeepIndices]
positive_label_indices = np.where(filtered_labels > 0)
filtered_bboxes = filtered_bboxes[positive_label_indices]
filtered_scores = filtered_scores[positive_label_indices]
filtered_labels = filtered_labels[positive_label_indices]
filtered_bboxes = filtered_bboxes[:, [1, 0, 3, 2]]
return [filtered_bboxes], [filtered_scores], [filtered_labels]
def compute_test_set_aps(eval_model, cfg):
num_test_images = cfg["DATA"].NUM_TEST_IMAGES
classes = cfg["DATA"].CLASSES
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT,
cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5),
dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
frcn_eval = eval_model(image_input, dims_input)
# Create the minibatch source
minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TEST_MAP_FILE,
cfg["DATA"].TEST_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=False,
use_flipping=False,
max_images=cfg["DATA"].NUM_TEST_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=None)
# define mapping from reader streams to network inputs
input_map = {
minibatch_source.image_si: image_input,
minibatch_source.roi_si: roi_input,
minibatch_source.dims_si: dims_input
}
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(num_test_images)]
for _ in range(cfg["DATA"].NUM_CLASSES)]
# evaluate test images and write netwrok output to file
print("Evaluating Faster R-CNN model for %s images." % num_test_images)
all_gt_infos = {key: [] for key in classes}
for img_i in range(0, num_test_images):
mb_data = minibatch_source.next_minibatch(1, input_map=input_map)
gt_row = mb_data[roi_input].asarray()
gt_row = gt_row.reshape((cfg.INPUT_ROIS_PER_IMAGE, 5))
all_gt_boxes = gt_row[np.where(gt_row[:, -1] > 0)]
for cls_index, cls_name in enumerate(classes):
if cls_index == 0: continue
cls_gt_boxes = all_gt_boxes[np.where(
all_gt_boxes[:, -1] == cls_index)]
all_gt_infos[cls_name].append({
'bbox':
np.array(cls_gt_boxes),
'difficult': [False] * len(cls_gt_boxes),
'det': [False] * len(cls_gt_boxes)
})
output = frcn_eval.eval({
image_input: mb_data[image_input],
dims_input: mb_data[dims_input]
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
mb_data[dims_input].asarray())
labels.shape = labels.shape + (1, )
scores.shape = scores.shape + (1, )
coords_score_label = np.hstack((regressed_rois, scores, labels))
# shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score
for cls_j in range(1, cfg["DATA"].NUM_CLASSES):
coords_score_label_for_cls = coords_score_label[np.where(
coords_score_label[:, -1] == cls_j)]
all_boxes[cls_j][
img_i] = coords_score_label_for_cls[:, :-1].astype(np.float32,
copy=False)
if (img_i + 1) % 100 == 0:
print("Processed {} samples".format(img_i + 1))
# calculate mAP
aps = evaluate_detections(all_boxes,
all_gt_infos,
classes,
use_gpu_nms=cfg.USE_GPU_NMS,
device_id=cfg.GPU_ID,
nms_threshold=cfg.RESULTS_NMS_THRESHOLD,
conf_threshold=cfg.RESULTS_NMS_CONF_THRESHOLD)
return aps
def compute_test_set_aps(eval_model, cfg):
num_test_images = cfg["DATA"].NUM_TEST_IMAGES
classes = cfg["DATA"].CLASSES
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
roi_proposals = input_variable((cfg.NUM_ROI_PROPOSALS, 4), dynamic_axes=[Axis.default_batch_axis()], name="roi_proposals")
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
frcn_eval = eval_model(image_input, roi_proposals)
# Create the minibatch source
if cfg.USE_PRECOMPUTED_PROPOSALS:
try:
cfg["DATA"].TEST_PRECOMPUTED_PROPOSALS_FILE = os.path.join(cfg["DATA"].MAP_FILE_PATH, cfg["DATA"].TEST_PRECOMPUTED_PROPOSALS_FILE)
proposal_provider = ProposalProvider.fromfile(cfg["DATA"].TEST_PRECOMPUTED_PROPOSALS_FILE, cfg.NUM_ROI_PROPOSALS)
except:
print("To use precomputed proposals please specify the following parameters in your configuration:\n"
"__C.DATA.TRAIN_PRECOMPUTED_PROPOSALS_FILE\n"
"__C.DATA.TEST_PRECOMPUTED_PROPOSALS_FILE")
exit(-1)
else:
proposal_provider = ProposalProvider.fromconfig(cfg)
minibatch_source = ObjectDetectionMinibatchSource(
cfg["DATA"].TEST_MAP_FILE,
cfg["DATA"].TEST_ROI_FILE,
max_annotations_per_image=cfg.INPUT_ROIS_PER_IMAGE,
pad_width=cfg.IMAGE_WIDTH,
pad_height=cfg.IMAGE_HEIGHT,
pad_value=cfg["MODEL"].IMG_PAD_COLOR,
randomize=False, use_flipping=False,
max_images=cfg["DATA"].NUM_TEST_IMAGES,
num_classes=cfg["DATA"].NUM_CLASSES,
proposal_provider=proposal_provider,
provide_targets=False)
# define mapping from reader streams to network inputs
input_map = {
minibatch_source.image_si: image_input,
minibatch_source.roi_si: roi_input,
minibatch_source.proposals_si: roi_proposals,
minibatch_source.dims_si: dims_input
}
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(num_test_images)] for _ in range(cfg["DATA"].NUM_CLASSES)]
# evaluate test images and write netwrok output to file
print("Evaluating Fast R-CNN model for %s images." % num_test_images)
all_gt_infos = {key: [] for key in classes}
for img_i in range(0, num_test_images):
mb_data = minibatch_source.next_minibatch(1, input_map=input_map)
gt_row = mb_data[roi_input].asarray()
gt_row = gt_row.reshape((cfg.INPUT_ROIS_PER_IMAGE, 5))
all_gt_boxes = gt_row[np.where(gt_row[:,-1] > 0)]
for cls_index, cls_name in enumerate(classes):
if cls_index == 0: continue
cls_gt_boxes = all_gt_boxes[np.where(all_gt_boxes[:,-1] == cls_index)]
all_gt_infos[cls_name].append({'bbox': np.array(cls_gt_boxes),
'difficult': [False] * len(cls_gt_boxes),
'det': [False] * len(cls_gt_boxes)})
output = frcn_eval.eval({image_input: mb_data[image_input], roi_proposals: mb_data[roi_proposals]})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = mb_data[roi_proposals].data.asarray()
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, mb_data[dims_input].asarray())
labels.shape = labels.shape + (1,)
scores.shape = scores.shape + (1,)
coords_score_label = np.hstack((regressed_rois, scores, labels))
# shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score
for cls_j in range(1, cfg["DATA"].NUM_CLASSES):
coords_score_label_for_cls = coords_score_label[np.where(coords_score_label[:,-1] == cls_j)]
all_boxes[cls_j][img_i] = coords_score_label_for_cls[:,:-1].astype(np.float32, copy=False)
if (img_i+1) % 100 == 0:
print("Processed {} samples".format(img_i+1))
# calculate mAP
aps = evaluate_detections(all_boxes, all_gt_infos, classes,
use_gpu_nms = cfg.USE_GPU_NMS,
device_id = cfg.GPU_ID,
nms_threshold=cfg.RESULTS_NMS_THRESHOLD,
conf_threshold = cfg.RESULTS_NMS_CONF_THRESHOLD)
return aps
def process_image(self, img_path):
out_cls_pred, out_rpn_rois, out_bbox_regr, dims = self.process_image_detailed(img_path)
labels = out_cls_pred.argmax(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
return regressed_rois, out_cls_pred
