def eval_and_plot_faster_rcnn(eval_model, num_images_to_plot, test_map_file, img_shape,
results_base_path, feature_node_name, classes,
drawUnregressedRois=False, drawNegativeRois=False,
nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8):
# get image paths
with open(test_map_file) as f:
content = f.readlines()
img_base_path = os.path.dirname(os.path.abspath(test_map_file))
img_file_names = [os.path.join(img_base_path, x.split('\t')[1]) for x in content]
# prepare model
image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
#dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6))
print("Plotting results from Faster R-CNN model for %s images." % num_images_to_plot)
for i in range(0, num_images_to_plot):
imgPath = img_file_names[i]
# evaluate single image
_, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1,) + dims_input.shape
output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
if drawUnregressedRois:
# plot results without final regression
imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave("{}/{}_{}".format(results_base_path, i, os.path.basename(imgPath)), imgDebug)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores,
nms_threshold=nmsThreshold,
conf_threshold=nmsConfThreshold)
img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=nmsKeepIndices,
boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave("{}/{}_regr_{}".format(results_base_path, i, os.path.basename(imgPath)), img)
def eval_and_plot_faster_rcnn(eval_model, num_images_to_plot, test_map_file, img_shape,
results_base_path, feature_node_name, classes,
drawUnregressedRois=False, drawNegativeRois=False,
nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8):
# get image paths
with open(test_map_file) as f:
content = f.readlines()
img_base_path = os.path.dirname(os.path.abspath(test_map_file))
img_file_names = [os.path.join(img_base_path, x.split('\t')[1]) for x in content]
# prepare model
image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
#dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6))
print("Plotting results from Faster R-CNN model for %s images." % num_images_to_plot)
for i in range(0, num_images_to_plot):
imgPath = img_file_names[i]
# evaluate single image
_, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1,) + dims_input.shape
output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
if drawUnregressedRois:
# plot results without final regression
imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave("{}/{}_{}".format(results_base_path, i, os.path.basename(imgPath)), imgDebug)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores,
nms_threshold=nmsThreshold,
conf_threshold=nmsConfThreshold)
img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=nmsKeepIndices,
boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave("{}/{}_regr_{}".format(results_base_path, i, os.path.basename(imgPath)), img)
def eval_faster_rcnn_mAP(eval_model):
img_map_file = globalvars['test_map_file']
roi_map_file = globalvars['test_roi_file']
classes = globalvars['classes']
image_input = input_variable((num_channels, image_height, image_width),
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5),
dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
frcn_eval = eval_model(image_input, dims_input)
# Create the minibatch source
minibatch_source = ObjectDetectionMinibatchSource(
img_map_file,
roi_map_file,
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width,
pad_height=image_height,
pad_value=img_pad_value,
randomize=False,
use_flipping=False,
max_images=cfg["CNTK"].NUM_TEST_IMAGES)
# define mapping from reader streams to network inputs
input_map = {
minibatch_source.image_si: image_input,
minibatch_source.roi_si: roi_input,
minibatch_source.dims_si: dims_input
}
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(num_test_images)]
for _ in range(globalvars['num_classes'])]
# evaluate test images and write netwrok output to file
print("Evaluating Faster R-CNN model for %s images." % num_test_images)
all_gt_infos = {key: [] for key in classes}
for img_i in range(0, num_test_images):
mb_data = minibatch_source.next_minibatch(1, input_map=input_map)
gt_row = mb_data[roi_input].asarray()
gt_row = gt_row.reshape((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5))
all_gt_boxes = gt_row[np.where(gt_row[:, -1] > 0)]
for cls_index, cls_name in enumerate(classes):
if cls_index == 0: continue
cls_gt_boxes = all_gt_boxes[np.where(
all_gt_boxes[:, -1] == cls_index)]
all_gt_infos[cls_name].append({
'bbox':
np.array(cls_gt_boxes),
'difficult': [False] * len(cls_gt_boxes),
'det': [False] * len(cls_gt_boxes)
})
output = frcn_eval.eval({
image_input: mb_data[image_input],
dims_input: mb_data[dims_input]
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
mb_data[dims_input].asarray())
labels.shape = labels.shape + (1, )
scores.shape = scores.shape + (1, )
coords_score_label = np.hstack((regressed_rois, scores, labels))
# shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score
for cls_j in range(1, globalvars['num_classes']):
coords_score_label_for_cls = coords_score_label[np.where(
coords_score_label[:, -1] == cls_j)]
all_boxes[cls_j][
img_i] = coords_score_label_for_cls[:, :-1].astype(np.float32,
copy=False)
if (img_i + 1) % 100 == 0:
print("Processed {} samples".format(img_i + 1))
confusions = None
try:
conf_file = cfg["CNTK"].CONFUSION_FILE
conf_file = os.path.join(map_file_path, conf_file)
confusions = confusions_map(classes, conf_file)
except:
confusions = None
# calculate mAP
aps, fp_errors = evaluate_detections(
all_boxes,
all_gt_infos,
classes,
nms_threshold=cfg["CNTK"].RESULTS_NMS_THRESHOLD,
conf_threshold=cfg["CNTK"].RESULTS_NMS_CONF_THRESHOLD,
soft=cfg["CNTK"].RESULTS_NMS_SOFT,
confusions=confusions)
if fp_errors:
output_file = os.path.join(
globalvars['output_path'], "{}_{}_fps.txt".format(
cfg["CNTK"].BASE_MODEL,
"e2e" if globalvars['train_e2e'] else "4stage"))
log_fp_errors(fp_errors, output_file)
ap_list = []
for class_name in aps:
ap_list += [aps[class_name]]
print('AP for {:>15} = {:.4f}'.format(class_name, aps[class_name]))
meanAP = np.nanmean(ap_list)
print('Mean AP = {:.4f}'.format(meanAP))
return meanAP
def faster_rcnn_pred(eval_model,
prediction_in,
prediction_out,
class_map_file,
img_shape,
feature_node_name,
nms_threshold=0.5,
nms_conf_threshold=0.0,
headers=True,
output_width_height=False,
suppressed_labels=()):
"""
Performs predictions with a model in continuous mode.
:param eval_model: the model to use
:param prediction_in: the input dir for images (list)
:param prediction_out: the output dir for images and ROI CSV files (list)
:param class_map_file: the class map file to use
:param img_shape: the shpage
:param feature_node_name: the name of the feature node
:param nms_threshold:
:param nms_conf_threshold:
:param headers: whether to output the headers in the ROI file
:param output_width_height: whether to output x/y/width/height or x0/y0/x1/y1
:param suppressed_labels: the labels to suppress from being output in the ROI files
"""
# load labels
str_labels = load_class_labels(class_map_file)
print("Class labels: %s" % str(str_labels))
# prepare model
image_input = input_variable(img_shape,
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
dims_input = input_variable((1, 6),
dynamic_axes=[Axis.default_batch_axis()],
name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
while True:
any = False
# iterate directory pairs
for p in range(len(prediction_in)):
pred_in = prediction_in[p]
pred_out = prediction_out[p]
# only png and jpg files
files = [
(pred_in + os.sep + x) for x in os.listdir(pred_in)
if (x.lower().endswith(".png") or x.lower().endswith(".jpg"))
]
for f in files:
start = datetime.now()
print(start, "-", f)
img_path = pred_out + os.sep + os.path.basename(f)
roi_path = pred_out + os.sep + os.path.splitext(
os.path.basename(f))[0] + ".csv"
cntk_img_input = None
try:
_, cntk_img_input, dims = load_resize_and_pad(
f, img_shape[2], img_shape[1])
except Exception as e:
print(str(e))
try:
# delete any existing old files in output dir
if os.path.exists(img_path):
try:
os.remove(img_path)
except:
print(
"Failed to remove existing image in output directory: ",
img_path)
if os.path.exists(roi_path):
try:
os.remove(roi_path)
except:
print(
"Failed to remove existing ROI file in output directory: ",
roi_path)
# move into output dir
os.rename(f, img_path)
except:
img_path = None
if cntk_img_input is None:
continue
if img_path is None:
continue
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1, ) + dims_input.shape
output = frcn_eval.eval({
frcn_eval.arguments[0]: [cntk_img_input],
frcn_eval.arguments[1]:
dims_input
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr,
labels, dims)
nms_keep_indices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
nms_threshold=nms_threshold,
conf_threshold=nms_conf_threshold)
save_rois_to_file(regressed_rois,
nms_keep_indices,
labels,
str_labels,
scores,
pred_out,
img_path,
headers=headers,
output_width_height=output_width_height,
suppressed_labels=suppressed_labels,
dims=dims)
timediff = datetime.now() - start
print(" time:", timediff)
# nothing processed at all, lets wait for files to appear
if not any:
sleep(1)
def faster_rcnn_eval(eval_model,
test_map_file,
class_map_file,
img_shape,
results_base_path,
feature_node_name,
nms_threshold=0.5,
nms_conf_threshold=0.0,
headers=True,
output_width_height=False,
suppressed_labels=()):
"""
Tests a Faster R-CNN model and outputs rois.
:param eval_model: the model
:param test_map_file: the map file with the test images/labels
:param class_map_file: the class map file
:param img_shape: the shape
:param results_base_path: the base directory for the results
:param feature_node_name: the feature node name
:param nms_threshold:
:param nms_conf_threshold:
:param headers: whether to output the headers in the ROI file
:param output_width_height: whether to output x/y/width/height or x0/y0/x1/y1
:param suppressed_labels: the labels to suppress from being output in the ROI files
"""
# load labels
str_labels = load_class_labels(class_map_file)
print("Class labels: %s" % str(str_labels))
# get image paths
with open(test_map_file) as f:
content = f.readlines()
img_base_path = os.path.dirname(os.path.abspath(test_map_file))
img_file_names = [
os.path.join(img_base_path,
x.split('\t')[1]) for x in content
]
# prepare model
image_input = input_variable(img_shape,
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
dims_input = input_variable((1, 6),
dynamic_axes=[Axis.default_batch_axis()],
name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
print("Outputting results from Faster R-CNN model for %s images." %
len(img_file_names))
for i in range(0, len(img_file_names)):
start = datetime.now()
img_path = img_file_names[i]
print(
str(i + 1) + "/" + str(len(img_file_names)) + ": " +
img_file_names[i])
# evaluate single image
_, cntk_img_input, dims = load_resize_and_pad(img_path, img_shape[2],
img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1, ) + dims_input.shape
output = frcn_eval.eval({
frcn_eval.arguments[0]: [cntk_img_input],
frcn_eval.arguments[1]: dims_input
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
dims)
nms_keep_indices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
nms_threshold=nms_threshold,
conf_threshold=nms_conf_threshold)
save_rois_to_file(regressed_rois,
nms_keep_indices,
labels,
str_labels,
scores,
results_base_path,
img_path,
headers=headers,
output_width_height=output_width_height,
suppressed_labels=suppressed_labels,
dims=dims)
timediff = datetime.now() - start
print(" time:", timediff)
def faster_rcnn_eval_and_plot(eval_model,
num_images_to_plot,
test_map_file,
class_map_file,
img_shape,
results_base_path,
feature_node_name,
classes,
draw_unregressed_rois=False,
draw_negative_rois=False,
nms_threshold=0.5,
nms_conf_threshold=0.0,
bgr_plot_threshold=0.8,
headers=True,
output_width_height=False,
suppressed_labels=()):
"""
Tests a Faster R-CNN model and plots images with detected boxes.
:param eval_model: the model to evaluate
:param num_images_to_plot: the number of images to plot
:param test_map_file: the map file with the test images/labels
:param class_map_file: the class map file
:param img_shape: the shape
:param results_base_path: the base of the output directory
:param feature_node_name: the feature node name
:param classes:
:param draw_unregressed_rois: whether to draw unregressed ROIs
:param draw_negative_rois: whether to draw negative ROIs (eg background)
:param nms_threshold:
:param nms_conf_threshold:
:param bgr_plot_threshold:
:param headers: whether to output the headers in the ROI file
:param output_width_height: whether to output x/y/width/height or x0/y0/x1/y1
"""
# load labels
str_labels = load_class_labels(class_map_file)
print("Class labels: %s" % str(str_labels))
# get image paths
with open(test_map_file) as f:
content = f.readlines()
img_base_path = os.path.dirname(os.path.abspath(test_map_file))
img_file_names = [
os.path.join(img_base_path,
x.split('\t')[1]) for x in content
]
# prepare model
image_input = input_variable(img_shape,
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
dims_input = input_variable((1, 6),
dynamic_axes=[Axis.default_batch_axis()],
name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
# dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6))
print("Plotting results from Faster R-CNN model for %s images." %
num_images_to_plot)
for i in range(0, num_images_to_plot):
img_path = img_file_names[i]
# save padded image
write_padded_image(
img_path, "{}/{}-padded.jpg".format(
results_base_path,
os.path.splitext(os.path.basename(img_path))[0]))
# evaluate single image
_, cntk_img_input, dims = load_resize_and_pad(img_path, img_shape[2],
img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1, ) + dims_input.shape
output = frcn_eval.eval({
frcn_eval.arguments[0]: [cntk_img_input],
frcn_eval.arguments[1]: dims_input
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
if draw_unregressed_rois:
# plot results without final regression
img_debug = faster_rcnn_visualize_results(
img_path,
labels,
scores,
out_rpn_rois,
img_shape[2],
img_shape[1],
classes,
nms_keep_indices=None,
draw_negative_rois=draw_negative_rois,
decision_threshold=bgr_plot_threshold)
imsave(
"{}/{}".format(results_base_path, os.path.basename(img_path)),
img_debug)
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
dims)
nms_keep_indices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
nms_threshold=nms_threshold,
conf_threshold=nms_conf_threshold)
save_rois_to_file(regressed_rois,
nms_keep_indices,
labels,
str_labels,
scores,
results_base_path,
img_path,
headers=headers,
output_width_height=output_width_height,
suppressed_labels=suppressed_labels,
dims=dims)
img = faster_rcnn_visualize_results(
img_path,
labels,
scores,
regressed_rois,
img_shape[2],
img_shape[1],
classes,
nms_keep_indices=nms_keep_indices,
draw_negative_rois=draw_negative_rois,
decision_threshold=bgr_plot_threshold)
imsave(
"{}/{}-regr.jpg".format(
results_base_path,
os.path.splitext(os.path.basename(img_path)))[0], img)
def eval_faster_rcnn(eval_model, imgPath, img_shape,
results_base_path, feature_node_name, classes, mode,
drawUnregressedRois=False, drawNegativeRois=False,
nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8):
# prepare model
image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
#dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6))
print("Plotting results from Faster R-CNN model for image.")
# evaluate single image
_, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1,) + dims_input.shape
output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
if mode=="returntags":
class Tag(object):
def __init__(self, label, score, bbox):
self.label = label
self.score = score
self.bbox = bbox
def serialize(self):
return {
'label': self.label,
'score': self.score,
'bbox': self.bbox,
}
results = []
for i in range(len(out_rpn_rois)):
if labels[i] != 0:
x = Tag(str(classes[labels[i]]), str(scores[i]), str(out_rpn_rois[i]))
results.append(x)
return results
elif mode=="returnimage":
evaluated_image_path = "{}/{}".format(results_base_path, 'evaluated_' + os.path.basename(imgPath))
if drawUnregressedRois:
# plot results without final regression
imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave(evaluated_image_path, imgDebug)
else:
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores,
nms_threshold=nmsThreshold,
conf_threshold=nmsConfThreshold)
img = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=nmsKeepIndices,
boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave(evaluated_image_path, img)
return evaluated_image_path
else:
raise ValueError("Unsupported value found in 'mode' parameter")
def eval_faster_rcnn(eval_model,
imgPath,
img_shape,
results_base_path,
feature_node_name,
classes,
mode,
drawUnregressedRois=False,
drawNegativeRois=False,
nmsThreshold=0.5,
nmsConfThreshold=0.0,
bgrPlotThreshold=0.8):
# prepare model
image_input = input_variable(img_shape,
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
dims_input = input_variable((1, 6),
dynamic_axes=[Axis.default_batch_axis()],
name='dims_input')
# if your model hasn't been loaded proper or in the proper place this is the line to look at. This line takes a model and loads it
# in preparation for the evaluation
try:
frcn_eval = eval_model(image_input, dims_input)
except:
raise TypeError("Loading existing model from %s" % model_path)
print("Plotting results from Faster R-CNN model for image.")
_, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2],
img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1, ) + dims_input.shape
output = frcn_eval.eval({
frcn_eval.arguments[0]: [cntk_img_input],
frcn_eval.arguments[1]: dims_input
})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
if mode == "returntags":
class Tag(object):
def __init__(self, label, score, bbox):
self.label = label
self.score = score
self.bbox = bbox
def serialize(self):
return {
'label': self.label,
'score': self.score,
'bbox': self.bbox,
}
results = []
for i in range(len(out_rpn_rois)):
if labels[i] != 0:
x = Tag(str(classes[labels[i]]), str(scores[i]),
str(out_rpn_rois[i]))
results.append(x)
return results
elif mode == "returnimage":
evaluated_image_path = "{}/{}".format(
results_base_path, 'evaluated_' + os.path.basename(imgPath))
if drawUnregressedRois:
# plot results without final regression
imgDebug = visualizeResultsFaster(
imgPath,
labels,
scores,
out_rpn_rois,
img_shape[2],
img_shape[1],
classes,
nmsKeepIndices=None,
boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave(evaluated_image_path, imgDebug)
else:
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels,
dims)
nmsKeepIndices = apply_nms_to_single_image_results(
regressed_rois,
labels,
scores,
nms_threshold=nmsThreshold,
conf_threshold=nmsConfThreshold)
img = visualizeResultsFaster(imgPath,
labels,
scores,
regressed_rois,
img_shape[2],
img_shape[1],
classes,
nmsKeepIndices=nmsKeepIndices,
boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave(evaluated_image_path, img)
return evaluated_image_path
else:
raise ValueError("Unsupported value found in 'mode' parameter")
def eval_faster_rcnn_mAP(eval_model):
img_map_file = globalvars['test_map_file']
roi_map_file = globalvars['test_roi_file']
classes = globalvars['classes']
image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
frcn_eval = eval_model(image_input, dims_input)
# Create the minibatch source
minibatch_source = ObjectDetectionMinibatchSource(
img_map_file, roi_map_file,
max_annotations_per_image=cfg["CNTK"].INPUT_ROIS_PER_IMAGE,
pad_width=image_width, pad_height=image_height, pad_value=img_pad_value,
randomize=False, use_flipping=False,
max_images=cfg["CNTK"].NUM_TEST_IMAGES)
# define mapping from reader streams to network inputs
input_map = {
minibatch_source.image_si: image_input,
minibatch_source.roi_si: roi_input,
minibatch_source.dims_si: dims_input
}
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(num_test_images)] for _ in range(globalvars['num_classes'])]
# evaluate test images and write netwrok output to file
print("Evaluating Faster R-CNN model for %s images." % num_test_images)
all_gt_infos = {key: [] for key in classes}
for img_i in range(0, num_test_images):
mb_data = minibatch_source.next_minibatch(1, input_map=input_map)
gt_row = mb_data[roi_input].asarray()
gt_row = gt_row.reshape((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5))
all_gt_boxes = gt_row[np.where(gt_row[:,-1] > 0)]
for cls_index, cls_name in enumerate(classes):
if cls_index == 0: continue
cls_gt_boxes = all_gt_boxes[np.where(all_gt_boxes[:,-1] == cls_index)]
all_gt_infos[cls_name].append({'bbox': np.array(cls_gt_boxes),
'difficult': [False] * len(cls_gt_boxes),
'det': [False] * len(cls_gt_boxes)})
output = frcn_eval.eval({image_input: mb_data[image_input], dims_input: mb_data[dims_input]})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1)
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, mb_data[dims_input].asarray())
labels.shape = labels.shape + (1,)
scores.shape = scores.shape + (1,)
coords_score_label = np.hstack((regressed_rois, scores, labels))
# shape of all_boxes: e.g. 21 classes x 4952 images x 58 rois x 5 coords+score
for cls_j in range(1, globalvars['num_classes']):
coords_score_label_for_cls = coords_score_label[np.where(coords_score_label[:,-1] == cls_j)]
all_boxes[cls_j][img_i] = coords_score_label_for_cls[:,:-1].astype(np.float32, copy=False)
if (img_i+1) % 100 == 0:
print("Processed {} samples".format(img_i+1))
# calculate mAP
aps = evaluate_detections(all_boxes, all_gt_infos, classes,
nms_threshold=cfg["CNTK"].RESULTS_NMS_THRESHOLD,
conf_threshold = cfg["CNTK"].RESULTS_NMS_CONF_THRESHOLD)
ap_list = []
for class_name in aps:
ap_list += [aps[class_name]]
print('AP for {:>15} = {:.4f}'.format(class_name, aps[class_name]))
meanAP = np.nanmean(ap_list)
print('Mean AP = {:.4f}'.format(meanAP))
return meanAP
def eval_faster_rcnn(eval_model, imgPath, img_shape,
results_base_path, feature_node_name, classes, mode,
drawUnregressedRois=False, drawNegativeRois=False,
nmsThreshold=0.5, nmsConfThreshold=0.0, bgrPlotThreshold = 0.8):
# prepare model
image_input = input_variable(img_shape, dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
dims_input = input_variable((1,6), dynamic_axes=[Axis.default_batch_axis()], name='dims_input')
frcn_eval = eval_model(image_input, dims_input)
#dims_input_const = cntk.constant([image_width, image_height, image_width, image_height, image_width, image_height], (1, 6))
print("Plotting results from Faster R-CNN model for image.")
# evaluate single image
_, cntk_img_input, dims = load_resize_and_pad(imgPath, img_shape[2], img_shape[1])
dims_input = np.array(dims, dtype=np.float32)
dims_input.shape = (1,) + dims_input.shape
output = frcn_eval.eval({frcn_eval.arguments[0]: [cntk_img_input], frcn_eval.arguments[1]: dims_input})
out_dict = dict([(k.name, k) for k in output])
out_cls_pred = output[out_dict['cls_pred']][0]
out_rpn_rois = output[out_dict['rpn_rois']][0]
out_bbox_regr = output[out_dict['bbox_regr']][0]
labels = out_cls_pred.argmax(axis=1)
scores = out_cls_pred.max(axis=1).tolist()
if mode=="returntags":
class Tag(object):
def __init__(self, label, score, bbox):
self.label = label
self.score = score
self.bbox = bbox
def serialize(self):
return {
'label': self.label,
'score': self.score,
'bbox': self.bbox,
}
results = []
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores,
nms_threshold=nmsThreshold,
conf_threshold=nmsConfThreshold)
print(len(out_rpn_rois))
imsave('./Temp/resized.jpg',imgDebug)
for i in range(len(out_rpn_rois)):
if labels[i] != 0:
x = Tag(str(classes[labels[i]]), str(scores[i]), str(out_rpn_rois[i]))
results.append(x)
# return {}
return results
elif mode=="returnimage":
evaluated_image_path = "{}/{}".format(results_base_path, 'evaluated_' + os.path.basename(imgPath))
if drawUnregressedRois:
# plot results without final regression
imgDebug = visualizeResultsFaster(imgPath, labels, scores, out_rpn_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=None, boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
imsave(evaluated_image_path, imgDebug)
else:
# apply regression and nms to bbox coordinates
regressed_rois = regress_rois(out_rpn_rois, out_bbox_regr, labels, dims)
nmsKeepIndices = apply_nms_to_single_image_results(regressed_rois, labels, scores,
nms_threshold=nmsThreshold,
conf_threshold=nmsConfThreshold)
img,allboxes = visualizeResultsFaster(imgPath, labels, scores, regressed_rois, img_shape[2], img_shape[1],
classes, nmsKeepIndices=nmsKeepIndices,
boDrawNegativeRois=drawNegativeRois,
decisionThreshold=bgrPlotThreshold)
# imsave(evaluated_image_path, img)
allboxes=np.array(allboxes)
# perform non-maximum suppression on the bounding boxes
pick = non_max_suppression_fast(allboxes, 0.6)
# print("[x] after applying non-maximum, %d bounding boxes" % (len(pick)))
black_bg = 0*np.ones_like(img)
# loop over the picked bounding boxes and extract each of the box
for (startX, startY, endX, endY) in pick:
roi=img[startY:endY,startX:endX]
black_bg[startY:endY,startX:endX]=roi
result = black_bg.copy()
print(black_bg.shape)
image = cv2.cvtColor(black_bg, cv2.COLOR_RGB2HSV)
lower = np.array([18, 0, 0])
upper = np.array([179, 255, 255])
mask = cv2.inRange(image, lower, upper)
result = cv2.bitwise_and(result,result, mask=mask)
lengthThroughRotatedRectangle=[]
lengthThroughManualCalculation=[]
# length Calculation through Rotated Rectangle
image=cv2.cvtColor(result,cv2.COLOR_BGR2GRAY)
ret,thresh = cv2.threshold(image,127,255,0)
working_image=thresh.copy()
result_img=thresh.copy()
working_image[:,:]=0
result_img[:,:]=0
kernel = np.ones((7,7), np.uint8)
for (startX, startY, endX, endY) in pick:
cord=(int(startX),int(startY),int(endX),int(endY))
working_image[:,:]=0
working_image[cord[1]:cord[3],cord[0]:cord[2]]=thresh[cord[1]:cord[3],cord[0]:cord[2]]
result_img[cord[1]:cord[3],cord[0]:cord[2]]=thresh[cord[1]:cord[3],cord[0]:cord[2]]
working_image=cv2.morphologyEx(working_image, cv2.MORPH_OPEN, kernel)
contours, hierarchy = cv2.findContours(working_image,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
if(len(contours)>0):
c = max(contours, key = cv2.contourArea)
rect = cv2.minAreaRect(c)
(x, y), (width, height), angle=rect
height=int(height)
width=int(width)
if width >height:
height=width
lengthThroughRotatedRectangle.append(height);
# length Calculation through Manual process
for (startX, startY, endX, endY) in pick:
cord=(int(startX),int(startY),int(endX),int(endY))
widthofRectangle=cord[2]-cord[0]
NonZeroPixels=[]
threshold=0.20;
height=0
width=0
for i in range(startY,endY):
row=thresh[i,startX:endX]
NonZeroPixelsInRow=np.count_nonzero(row)
WidthRatioInRow=NonZeroPixelsInRow/widthofRectangle
if(WidthRatioInRow>threshold):
height=height+1
NonZeroPixels.append(NonZeroPixelsInRow)
width=round(sum(NonZeroPixels)/len(NonZeroPixels))
if(width > height):
height=width
lengthThroughManualCalculation.append(height)
# print("length through rotatedRectangle\n",lengthThroughRotatedRectangle)
# print("length through ManualCalculation\n",lengthThroughManualCalculation)
SpikesLength=[int((a+b)/2) for a,b in zip(lengthThroughRotatedRectangle,lengthThroughManualCalculation)]
# print("spike length\n",SpikesLength)
SpikesLength = [i * 0.1 for i in SpikesLength]
print("Spike Lenght in cm",SpikesLength)
# imsave(evaluated_image_path, thresh)
return SpikesLength
else:
raise ValueError("Unsupported value found in 'mode' parameter")