def display_differences(image,
gt_box, gt_class_id, gt_mask,
pred_box, pred_class_id, pred_score, pred_mask,
class_names, title="", ax=None,
show_mask=True, show_box=True,
iou_threshold=0.5, score_threshold=0.5):
"""Display ground truth and prediction instances on the same image."""
# Match predictions to ground truth
gt_match, pred_match, overlaps = utils.compute_matches(
gt_box, gt_class_id, gt_mask,
pred_box, pred_class_id, pred_score, pred_mask,
iou_threshold=iou_threshold, score_threshold=score_threshold)
# Ground truth = green. Predictions = red
colors = [(0, 1, 0, .8)] * len(gt_match)\
+ [(1, 0, 0, 1)] * len(pred_match)
# Concatenate GT and predictions
class_ids = np.concatenate([gt_class_id, pred_class_id])
scores = np.concatenate([np.zeros([len(gt_match)]), pred_score])
boxes = np.concatenate([gt_box, pred_box])
masks = np.concatenate([gt_mask, pred_mask], axis=-1)
# Captions per instance show score/IoU
captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format(
pred_score[i],
(overlaps[i, int(pred_match[i])]
if pred_match[i] > -1 else overlaps[i].max()))
for i in range(len(pred_match))]
# Set title if not provided
title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU"
# Display
display_instances(
image,
boxes, masks, class_ids,
class_names, scores, ax=ax,
show_bbox=show_box, show_mask=show_mask,
colors=colors, captions=captions,
title=title)
def display_differences(image,
gt_box, gt_class_id, gt_mask,
pred_box, pred_class_id, pred_score, pred_mask,
class_names, title="", ax=None,
show_mask=True, show_box=True,
iou_threshold=0.5, score_threshold=0.5):
"""Display ground truth and prediction instances on the same image."""
# Match predictions to ground truth
gt_match, pred_match, overlaps = utils.compute_matches(
gt_box, gt_class_id, gt_mask,
pred_box, pred_class_id, pred_score, pred_mask,
iou_threshold=iou_threshold, score_threshold=score_threshold)
# Ground truth = green. Predictions = red
colors = [(0, 1, 0, .8)] * len(gt_match)\
+ [(1, 0, 0, 1)] * len(pred_match)
# Concatenate GT and predictions
class_ids = np.concatenate([gt_class_id, pred_class_id])
scores = np.concatenate([np.zeros([len(gt_match)]), pred_score])
boxes = np.concatenate([gt_box, pred_box])
masks = np.concatenate([gt_mask, pred_mask], axis=-1)
# Captions per instance show score/IoU
captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format(
pred_score[i],
(overlaps[i, int(pred_match[i])]
if pred_match[i] > -1 else overlaps[i].max()))
for i in range(len(pred_match))]
# Set title if not provided
title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU"
# Display
display_instances(
image,
boxes, masks, class_ids,
class_names, scores, ax=ax,
show_bbox=show_box, show_mask=show_mask,
colors=colors, captions=captions,
title=title)
def banana_compute_ap(gt_boxes, gt_class_ids, gt_masks, pred_boxes,
pred_class_ids, pred_scores, pred_masks):
gt_match, pred_match, overlaps = utils.compute_matches(
gt_boxes, gt_class_ids, gt_masks, pred_boxes, pred_class_ids,
pred_scores, pred_masks)
# compute AP for special banana case
# clean redundant detectio
detect = 0
for i, o in enumerate(overlaps):
if detect == 1 and o > 0.5:
overlaps = np.delete(overlaps, i)
pred_match = np.delete(pred_match, i)
gt_match - -1
if o > 0.5:
detect = 1
# Compute precision and recall at each prediction box step
precisions = np.cumsum(pred_match > -1) / (np.arange(len(pred_match)) + 1)
recalls = np.cumsum(pred_match > -1).astype(np.float32) / len(gt_match)
# Pad with start and end values to simplify the math
precisions = np.concatenate([[0], precisions, [0]])
recalls = np.concatenate([[0], recalls, [1]])
# Ensure precision values decrease but don't increase. This way, the
# precision value at each recall threshold is the maximum it can be
# for all following recall thresholds, as specified by the VOC paper.
for i in range(len(precisions) - 2, -1, -1):
precisions[i] = np.maximum(precisions[i], precisions[i + 1])
# Compute mean AP over recall range
indices = np.where(recalls[:-1] != recalls[1:])[0] + 1
mAP = np.sum(
(recalls[indices] - recalls[indices - 1]) * precisions[indices])
return mAP
"""
def compare_instance_masks(model, dataset, config, image_ids, output_dir, iou_threshold=0.5, score_threshold=0.5,
show_box=False, show_mask=True, ax=None, title=None):
class_names = dataset.class_names
for image_id in image_ids:
image_id = int(image_id)
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id,
dataset.image_reference(image_id)))
# Run object detection
results = model.detect([image], verbose=1)[0]
pred_bbox = results['rois']
pred_class_id = results['class_ids']
pred_score = results['scores']
pred_mask = results['masks']
"""Display ground truth and prediction instances on the same image."""
# Match predictions to ground truth
gt_match, pred_match, overlaps = utils.compute_matches(
gt_bbox, gt_class_id, gt_mask,
pred_bbox, pred_class_id, pred_score, pred_mask,
iou_threshold=iou_threshold, score_threshold=score_threshold)
# Ground truth = green. Predictions = red
colors = [(0.686, 0.35, 0.18, .5)] * len(gt_match) \
+ [(0.471, 0.558, 0.28, 1)] * len(pred_match)
# Iterate over the GT and extract difference images between prediction and groundtruth.
gt_mask_zeros = np.zeros_like(gt_mask)
pred_mask_zeros = np.zeros_like(pred_mask)
for gt_index in range(len(gt_match)):
pred_index = int(gt_match[gt_index])
# There was a match between prediction and ground truth
if pred_index != -1:
gt_inst_mask = gt_mask[:, :, gt_index]
gt_mask_zeros[:, :, gt_index] = gt_inst_mask
pred_inst_mask = pred_mask[:, :, pred_index]
diff = np.abs(gt_inst_mask.astype(np.int) - pred_inst_mask.astype(np.int))
pred_mask_zeros[:, :, pred_index] = diff
else:
gt_mask_zeros[:, :, gt_index] = np.zeros_like(gt_mask[:, :, gt_index])
gt_mask = gt_mask_zeros
pred_mask = pred_mask_zeros
# Concatenate GT and predictions -> don't concatenate them but determine the respective differences
class_ids = np.concatenate([gt_class_id, pred_class_id])
scores = np.concatenate([np.zeros([len(gt_match)]), pred_score])
boxes = np.concatenate([gt_bbox, pred_bbox])
masks = np.concatenate([gt_mask, pred_mask], axis=-1)
# Captions per instance show score/IoU
captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format(
pred_score[i],
(overlaps[i, int(pred_match[i])]
if pred_match[i] > -1 else overlaps[i].max()))
for i in range(len(pred_match))]
# Set title if not provided
title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU"
# Display
fig = visualize.display_instances(
image,
boxes, masks, class_ids,
class_names, scores, ax=ax,
show_bbox=show_box, show_mask=show_mask,
colors=colors, captions=None,
title=title)
output_path = os.path.join(output_dir, "instance_masks_diff_to_gt_id_{}.png".format(image_id))
print('Saving image {} to {}'.format(image_id, output_path))
fig.savefig(output_path)
def evaluate_model(self):
APs = list()
precisions_dict = {}
recall_dict = {}
true_positives_total = [0] * len(
self.dataset.class_info
) # array with true positives for each class
detections_total = [0] * len(
self.dataset.class_info) # detections for each class
gt_instances_total = [0] * len(
self.dataset.class_info) # gt_instances for each class
for index, image_id in enumerate(self.dataset.image_ids):
print('\n')
print("Processing image:id ", image_id)
# load image, bounding boxes and masks for the image id
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
self.dataset, self.cfg, image_id)
print("GT_Class_ID:")
print(gt_class_id)
print("image_meta:")
print(image_meta)
# convert pixel values (e.g. center)
#scaled_image = modellib.mold_image(image, self.cfg)
# convert image into one sample
sample = np.expand_dims(image, 0)
# print(len(image))
# make prediction
yhat = self.model.detect(sample, verbose=0)
# extract results for first sample
r = yhat[0]
# calculate statistics, including AP for each class
for items in self.dataset.class_info:
class_name = items["name"]
if class_name == "BG":
continue
class_id = items["id"]
gt_match, pred_match, overlaps = compute_matches(
gt_bbox, gt_class_id, gt_mask, r["rois"],
r["class_ids"], r["scores"], r['masks'], class_id)
AP, precisions, recalls, _ = compute_ap(
gt_bbox, gt_class_id, gt_mask, r["rois"],
r["class_ids"], r["scores"], r['masks'], class_id)
#precision, recall, f1 = compute_f1(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], class_id)
#visualize.display_instances(image[..., :3], r['rois'], r['masks'], r['class_ids'],
# dataset_val.class_names, r['scores'], figsize=(8,8), savepath=image_id) # added save option
true_positives = sum(i > -1 for i in pred_match)
detections = len(pred_match)
gt_instances = len(gt_match)
true_positives_total[class_id] = true_positives_total[
class_id] + true_positives
detections_total[
class_id] = detections_total[class_id] + detections
gt_instances_total[
class_id] = gt_instances_total[class_id] + gt_instances
print("---- " + class_name + " ----")
print("gt_instances: ", gt_instances)
print("true positives: ", true_positives)
print("detections: ", detections)
print("true_positives_total: ",
true_positives_total[class_id])
print("detections_total: {}", detections_total[class_id])
precisions_dict[image_id] = np.mean(precisions)
recall_dict[image_id] = np.mean(recalls)
# store
#APs.append(AP)
# calculate the mean AP, precision, recall, f1 across all images
#mAP = np.mean(APs)
for items in self.dataset.class_info:
class_name = items["name"]
if class_name == "BG":
continue
class_id = items["id"]
precision = true_positives_total[class_id] / detections_total[
class_id]
recall = true_positives_total[class_id] / gt_instances_total[
class_id]
print(class_name + " :")
#print("mAp: ", mAP)
print("precision: ", precision)
print("recall: ", recall)
return 0 # mAP,precisions_dict,recall_dict
def evaluate_model(self):
APs = list()
precisions_dict = {}
recall_dict = {}
true_positives_total = [0]*len(self.dataset.class_info) # array with true positives for each class
detections_total = [0]*len(self.dataset.class_info) # detections for each class
gt_instances_total = [0]*len(self.dataset.class_info) # gt_instances for each class
squared_errors_total = [[]]*len(self.dataset.class_info) # squared score error for each class
gt_scores_total = [[]]*len(self.dataset.class_info) # all gt_scores for each class
pred_scores_total = [[]]*len(self.dataset.class_info) # all predicted scores for each class
true_score_classes = [0]*len(self.dataset.class_info) # number of correct score classes (1-3) for each class
for index,image_id in enumerate(self.dataset.image_ids):
print('\n')
print("Processing image:id ",image_id)
# load image, bounding boxes and masks for the image id
image, image_meta, gt_class_id, gt_bbox, gt_mask, gt_score = modellib.load_image_gt(self.dataset, self.cfg,image_id,use_depth=False)
sample = np.expand_dims(image, 0)
# print(len(image))
# make prediction
yhat = self.model.detect(sample, verbose=1)
# extract results for first sample
r = yhat[0]
# calculate statistics, including AP for each class
for items in self.dataset.class_info:
if items["id"] >= 1: # do not perform calculations for background class
class_name = items["name"]
class_id = items["id"]
print("Class_id: ", r["class_ids"])
print("Props: ", r["class_scores"])
print("Feather_Scores: ", r["scores"])
gt_match, pred_match, overlaps = compute_matches(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["class_scores"], r['masks'], class_id)
AP, precisions, recalls, _ = compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["class_scores"], r['masks'], class_id)
# Calculate Score Error
pred_match_indices = pred_match[(pred_match>-1)].astype(int) # For each correctly predicted box it has the index of the matched gt box.
gt_match_indices = gt_match[(gt_match>-1)].astype(int) # for each correctly matched gt box it has the index of the matched predicted box
print("gt_score:", gt_score)
gt_match_scores = gt_score[pred_match_indices] # gt scores of each correctly predicted box
print("gt_match_scores: ", gt_match_scores)
print("pred_scores: ", r["scores"])
positive_score_indices = np.where(gt_match_scores>-1) # indices of positive gt scores
gt_positive_scores = gt_match_scores[positive_score_indices]
pred_positive_scores = r["scores"][positive_score_indices]
print("positive_gt_scores: ", gt_positive_scores)
print("positive_pred_scores: ", pred_positive_scores)
gt_scores_total[class_id] = np.concatenate((gt_scores_total[class_id], gt_positive_scores))
pred_scores_total[class_id] = np.concatenate((pred_scores_total[class_id], pred_positive_scores))
error = np.absolute(np.subtract(gt_positive_scores,pred_positive_scores))
squared_error = np.square(error)
squared_error_image = np.sum(squared_error)
print("MSE image: ", squared_error_image/len(positive_score_indices))
squared_errors_total[class_id] = np.concatenate((squared_errors_total[class_id], squared_error))
print("Current MSE: ", np.mean(squared_errors_total[class_id]))
#categorize scores in 3 classes and evaluate accuracy:
correct_score_class_count = len(np.where((np.round(gt_positive_scores)-np.round(pred_positive_scores))==0)[0])
print("Correct Score Classes:", correct_score_class_count)
true_score_classes[class_id] = true_score_classes[class_id] + correct_score_class_count
if len(squared_errors_total[class_id]) > 0:
score_accuracy = true_score_classes[class_id]/len(squared_errors_total[class_id])
else:
score_accuracy = 0
print("Current Score Accuracy: ", score_accuracy)
# calculate and sum up true positives and detections
true_positives = sum(i> -1 for i in pred_match)
detections = len(pred_match)
gt_instances = len(gt_match)
true_positives_total[class_id] = true_positives_total[class_id] + true_positives
detections_total[class_id] = detections_total[class_id] + detections
gt_instances_total[class_id] = gt_instances_total[class_id] + gt_instances
print("---- " + class_name + " ----")
print("gt_instances: ", gt_instances)
print("true positives: ", true_positives)
print("detections: ", detections)
print("true_positives_total: ", true_positives_total[class_id])
print("detections_total: {}", detections_total[class_id])
#print("(Predicted) Scores:" , gt_scores_total, pred_scores_total)
precisions_dict[image_id] = np.mean(precisions)
recall_dict[image_id] = np.mean(recalls)
# store
APs.append(AP)
#title = str(image_id) + 'no_mask'
visualize.display_instances(image[..., :3], r['rois'], r['masks'], r['class_ids'],
dataset_val.class_names, r['class_scores'], r['scores'], figsize=(8,8), show_mask=False, savepath=image_id) # added scores $
#visualize.display_differences(image[..., :3], gt_bbox, gt_class_id, gt_mask, gt_score, r['rois'], r['class_ids'], r['class_scores'],
# r['scores'], r['masks'], dataset_val.class_names,class_id, show_mask=False, savepath=image_id)
#Analyze activations
import tensorflow as tf
activations = self.model.run_graph([image], [
("input_image", tf.identity(self.model.keras_model.get_layer("input_image").output)),
("conv1", self.model.keras_model.get_layer("conv1").output),
("res2c_out", self.model.keras_model.get_layer("res2c_out").output),
("res3c_out", self.model.keras_model.get_layer("res3c_out").output),
("fpn_p4", self.model.keras_model.get_layer("fpn_p4").output),
("rpn_bbox", self.model.keras_model.get_layer("rpn_bbox").output),
("roi", self.model.keras_model.get_layer("ROI").output),
])
conv_sum = activations["conv1"][0,:,:,:].mean(axis=2)
conv_sum=np.expand_dims(conv_sum, axis=2)
res2c_sum = activations["res2c_out"][0,:,:,:].mean(axis=2)
res2c_sum=np.expand_dims(res2c_sum, axis=2)
print(activations["conv1"][0,:,:,:].shape)
print(conv_sum.shape)
#visualize.display_images(np.transpose(activations["input_image"][0,:,:,:4], [2, 0, 1]),titles=[image_id,2,3,4,5,6])
#visualize.display_images(np.transpose(activations["conv1"][0,:,:,:], [2, 0, 1]),titles=["Conv1", image_id, 3, 4,5,6,7,8,9,10])
#visualize.display_images(np.transpose(conv_sum, [2, 0, 1]),titles=["Conv1_Sum", image_id])
#visualize.display_images(np.transpose(res2c_sum, [2, 0, 1]),titles=["Res2C_Sum", image_id])
#visualize.display_images(np.transpose(activations["res3c_out"][0,:,:,:5], [2, 0, 1]),titles=["RES3C", image_id,3 ,4,5,6,7,8,9,10])
# calculate the mean AP, precision, recall, f1, MSE across all images
mAP = np.mean(APs)
for items in self.dataset.class_info:
class_name = items["name"]
if class_name != "Hen":
continue
class_id = items["id"]
precision = true_positives_total[class_id]/detections_total[class_id]
recall = true_positives_total[class_id]/gt_instances_total[class_id]
MSE = np.mean(squared_errors_total[class_id])
print(class_name + " :")
print("mAp: ", mAP)
print("precision: ", precision)
print("recall: ", recall)
print("MSE: ", MSE)
np.savetxt(f, gt_scores_total[class_id], delimiter=',')
np.savetxt(f, pred_scores_total[class_id], delimiter=',')
f.write('Precision: {0} Recall: {1} MSE: {2}\n'.format(precision, recall, MSE))
return 0
# If high confident skip
if r['scores'][j] >= 0.8:
continue
box = r['rois'][j:j+1]
class_id = r['class_ids'][j:j+1]
score = r['scores'][j:j+1]
mask = r['masks'][:,:,j:j+1]
found = False
# Check appear before
for k in range(1, 6):
if i - k < 0:
break
_, _r = preds[i-k]
gt_match, _, _ = utils.compute_matches(box, class_id, mask,
_r['rois'], _r['class_ids'], _r['scores'], _r['masks'],
iou_threshold=0.1, score_threshold=0.0)
if gt_match[0] != -1 and _r['scores'][int(gt_match[0])] > 0.7:
found = True
break
# Check appear after
for k in range(1, 6):
if i + k >= len(preds):
break
_, _r = preds[i+k]
gt_match, _, _ = utils.compute_matches(box, class_id, mask,
_r['rois'], _r['class_ids'], _r['scores'], _r['masks'],
iou_threshold=0.1, score_threshold=0.0)
if gt_match[0] != -1 and _r['scores'][int(gt_match[0])] > 0.7:
