def display_differences(image, gt_box, gt_class_id, gt_mask, pred_box, pred_class_id, pred_score, pred_mask, class_names, title="", ax=None, show_mask=True, show_box=True, iou_threshold=0.5, score_threshold=0.5): """Display ground truth and prediction instances on the same image.""" # Match predictions to ground truth gt_match, pred_match, overlaps = utils.compute_matches( gt_box, gt_class_id, gt_mask, pred_box, pred_class_id, pred_score, pred_mask, iou_threshold=iou_threshold, score_threshold=score_threshold) # Ground truth = green. Predictions = red colors = [(0, 1, 0, .8)] * len(gt_match)\ + [(1, 0, 0, 1)] * len(pred_match) # Concatenate GT and predictions class_ids = np.concatenate([gt_class_id, pred_class_id]) scores = np.concatenate([np.zeros([len(gt_match)]), pred_score]) boxes = np.concatenate([gt_box, pred_box]) masks = np.concatenate([gt_mask, pred_mask], axis=-1) # Captions per instance show score/IoU captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format( pred_score[i], (overlaps[i, int(pred_match[i])] if pred_match[i] > -1 else overlaps[i].max())) for i in range(len(pred_match))] # Set title if not provided title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU" # Display display_instances( image, boxes, masks, class_ids, class_names, scores, ax=ax, show_bbox=show_box, show_mask=show_mask, colors=colors, captions=captions, title=title)
def banana_compute_ap(gt_boxes, gt_class_ids, gt_masks, pred_boxes, pred_class_ids, pred_scores, pred_masks): gt_match, pred_match, overlaps = utils.compute_matches( gt_boxes, gt_class_ids, gt_masks, pred_boxes, pred_class_ids, pred_scores, pred_masks) # compute AP for special banana case # clean redundant detectio detect = 0 for i, o in enumerate(overlaps): if detect == 1 and o > 0.5: overlaps = np.delete(overlaps, i) pred_match = np.delete(pred_match, i) gt_match - -1 if o > 0.5: detect = 1 # Compute precision and recall at each prediction box step precisions = np.cumsum(pred_match > -1) / (np.arange(len(pred_match)) + 1) recalls = np.cumsum(pred_match > -1).astype(np.float32) / len(gt_match) # Pad with start and end values to simplify the math precisions = np.concatenate([[0], precisions, [0]]) recalls = np.concatenate([[0], recalls, [1]]) # Ensure precision values decrease but don't increase. This way, the # precision value at each recall threshold is the maximum it can be # for all following recall thresholds, as specified by the VOC paper. for i in range(len(precisions) - 2, -1, -1): precisions[i] = np.maximum(precisions[i], precisions[i + 1]) # Compute mean AP over recall range indices = np.where(recalls[:-1] != recalls[1:])[0] + 1 mAP = np.sum( (recalls[indices] - recalls[indices - 1]) * precisions[indices]) return mAP
"""
def compare_instance_masks(model, dataset, config, image_ids, output_dir, iou_threshold=0.5, score_threshold=0.5, show_box=False, show_mask=True, ax=None, title=None): class_names = dataset.class_names for image_id in image_ids: image_id = int(image_id) image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) info = dataset.image_info[image_id] print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id, dataset.image_reference(image_id))) # Run object detection results = model.detect([image], verbose=1)[0] pred_bbox = results['rois'] pred_class_id = results['class_ids'] pred_score = results['scores'] pred_mask = results['masks'] """Display ground truth and prediction instances on the same image.""" # Match predictions to ground truth gt_match, pred_match, overlaps = utils.compute_matches( gt_bbox, gt_class_id, gt_mask, pred_bbox, pred_class_id, pred_score, pred_mask, iou_threshold=iou_threshold, score_threshold=score_threshold) # Ground truth = green. Predictions = red colors = [(0.686, 0.35, 0.18, .5)] * len(gt_match) \ + [(0.471, 0.558, 0.28, 1)] * len(pred_match) # Iterate over the GT and extract difference images between prediction and groundtruth. gt_mask_zeros = np.zeros_like(gt_mask) pred_mask_zeros = np.zeros_like(pred_mask) for gt_index in range(len(gt_match)): pred_index = int(gt_match[gt_index]) # There was a match between prediction and ground truth if pred_index != -1: gt_inst_mask = gt_mask[:, :, gt_index] gt_mask_zeros[:, :, gt_index] = gt_inst_mask pred_inst_mask = pred_mask[:, :, pred_index] diff = np.abs(gt_inst_mask.astype(np.int) - pred_inst_mask.astype(np.int)) pred_mask_zeros[:, :, pred_index] = diff else: gt_mask_zeros[:, :, gt_index] = np.zeros_like(gt_mask[:, :, gt_index]) gt_mask = gt_mask_zeros pred_mask = pred_mask_zeros # Concatenate GT and predictions -> don't concatenate them but determine the respective differences class_ids = np.concatenate([gt_class_id, pred_class_id]) scores = np.concatenate([np.zeros([len(gt_match)]), pred_score]) boxes = np.concatenate([gt_bbox, pred_bbox]) masks = np.concatenate([gt_mask, pred_mask], axis=-1) # Captions per instance show score/IoU captions = ["" for m in gt_match] + ["{:.2f} / {:.2f}".format( pred_score[i], (overlaps[i, int(pred_match[i])] if pred_match[i] > -1 else overlaps[i].max())) for i in range(len(pred_match))] # Set title if not provided title = title or "Ground Truth and Detections\n GT=green, pred=red, captions: score/IoU" # Display fig = visualize.display_instances( image, boxes, masks, class_ids, class_names, scores, ax=ax, show_bbox=show_box, show_mask=show_mask, colors=colors, captions=None, title=title) output_path = os.path.join(output_dir, "instance_masks_diff_to_gt_id_{}.png".format(image_id)) print('Saving image {} to {}'.format(image_id, output_path)) fig.savefig(output_path)
def evaluate_model(self): APs = list() precisions_dict = {} recall_dict = {} true_positives_total = [0] * len( self.dataset.class_info ) # array with true positives for each class detections_total = [0] * len( self.dataset.class_info) # detections for each class gt_instances_total = [0] * len( self.dataset.class_info) # gt_instances for each class for index, image_id in enumerate(self.dataset.image_ids): print('\n') print("Processing image:id ", image_id) # load image, bounding boxes and masks for the image id image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( self.dataset, self.cfg, image_id) print("GT_Class_ID:") print(gt_class_id) print("image_meta:") print(image_meta) # convert pixel values (e.g. center) #scaled_image = modellib.mold_image(image, self.cfg) # convert image into one sample sample = np.expand_dims(image, 0) # print(len(image)) # make prediction yhat = self.model.detect(sample, verbose=0) # extract results for first sample r = yhat[0] # calculate statistics, including AP for each class for items in self.dataset.class_info: class_name = items["name"] if class_name == "BG": continue class_id = items["id"] gt_match, pred_match, overlaps = compute_matches( gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], class_id) AP, precisions, recalls, _ = compute_ap( gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], class_id) #precision, recall, f1 = compute_f1(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], class_id) #visualize.display_instances(image[..., :3], r['rois'], r['masks'], r['class_ids'], # dataset_val.class_names, r['scores'], figsize=(8,8), savepath=image_id) # added save option true_positives = sum(i > -1 for i in pred_match) detections = len(pred_match) gt_instances = len(gt_match) true_positives_total[class_id] = true_positives_total[ class_id] + true_positives detections_total[ class_id] = detections_total[class_id] + detections gt_instances_total[ class_id] = gt_instances_total[class_id] + gt_instances print("---- " + class_name + " ----") print("gt_instances: ", gt_instances) print("true positives: ", true_positives) print("detections: ", detections) print("true_positives_total: ", true_positives_total[class_id]) print("detections_total: {}", detections_total[class_id]) precisions_dict[image_id] = np.mean(precisions) recall_dict[image_id] = np.mean(recalls) # store #APs.append(AP) # calculate the mean AP, precision, recall, f1 across all images #mAP = np.mean(APs) for items in self.dataset.class_info: class_name = items["name"] if class_name == "BG": continue class_id = items["id"] precision = true_positives_total[class_id] / detections_total[ class_id] recall = true_positives_total[class_id] / gt_instances_total[ class_id] print(class_name + " :") #print("mAp: ", mAP) print("precision: ", precision) print("recall: ", recall) return 0 # mAP,precisions_dict,recall_dict
def evaluate_model(self): APs = list() precisions_dict = {} recall_dict = {} true_positives_total = [0]*len(self.dataset.class_info) # array with true positives for each class detections_total = [0]*len(self.dataset.class_info) # detections for each class gt_instances_total = [0]*len(self.dataset.class_info) # gt_instances for each class squared_errors_total = [[]]*len(self.dataset.class_info) # squared score error for each class gt_scores_total = [[]]*len(self.dataset.class_info) # all gt_scores for each class pred_scores_total = [[]]*len(self.dataset.class_info) # all predicted scores for each class true_score_classes = [0]*len(self.dataset.class_info) # number of correct score classes (1-3) for each class for index,image_id in enumerate(self.dataset.image_ids): print('\n') print("Processing image:id ",image_id) # load image, bounding boxes and masks for the image id image, image_meta, gt_class_id, gt_bbox, gt_mask, gt_score = modellib.load_image_gt(self.dataset, self.cfg,image_id,use_depth=False) sample = np.expand_dims(image, 0) # print(len(image)) # make prediction yhat = self.model.detect(sample, verbose=1) # extract results for first sample r = yhat[0] # calculate statistics, including AP for each class for items in self.dataset.class_info: if items["id"] >= 1: # do not perform calculations for background class class_name = items["name"] class_id = items["id"] print("Class_id: ", r["class_ids"]) print("Props: ", r["class_scores"]) print("Feather_Scores: ", r["scores"]) gt_match, pred_match, overlaps = compute_matches(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["class_scores"], r['masks'], class_id) AP, precisions, recalls, _ = compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["class_scores"], r['masks'], class_id) # Calculate Score Error pred_match_indices = pred_match[(pred_match>-1)].astype(int) # For each correctly predicted box it has the index of the matched gt box. gt_match_indices = gt_match[(gt_match>-1)].astype(int) # for each correctly matched gt box it has the index of the matched predicted box print("gt_score:", gt_score) gt_match_scores = gt_score[pred_match_indices] # gt scores of each correctly predicted box print("gt_match_scores: ", gt_match_scores) print("pred_scores: ", r["scores"]) positive_score_indices = np.where(gt_match_scores>-1) # indices of positive gt scores gt_positive_scores = gt_match_scores[positive_score_indices] pred_positive_scores = r["scores"][positive_score_indices] print("positive_gt_scores: ", gt_positive_scores) print("positive_pred_scores: ", pred_positive_scores) gt_scores_total[class_id] = np.concatenate((gt_scores_total[class_id], gt_positive_scores)) pred_scores_total[class_id] = np.concatenate((pred_scores_total[class_id], pred_positive_scores)) error = np.absolute(np.subtract(gt_positive_scores,pred_positive_scores)) squared_error = np.square(error) squared_error_image = np.sum(squared_error) print("MSE image: ", squared_error_image/len(positive_score_indices)) squared_errors_total[class_id] = np.concatenate((squared_errors_total[class_id], squared_error)) print("Current MSE: ", np.mean(squared_errors_total[class_id])) #categorize scores in 3 classes and evaluate accuracy: correct_score_class_count = len(np.where((np.round(gt_positive_scores)-np.round(pred_positive_scores))==0)[0]) print("Correct Score Classes:", correct_score_class_count) true_score_classes[class_id] = true_score_classes[class_id] + correct_score_class_count if len(squared_errors_total[class_id]) > 0: score_accuracy = true_score_classes[class_id]/len(squared_errors_total[class_id]) else: score_accuracy = 0 print("Current Score Accuracy: ", score_accuracy) # calculate and sum up true positives and detections true_positives = sum(i> -1 for i in pred_match) detections = len(pred_match) gt_instances = len(gt_match) true_positives_total[class_id] = true_positives_total[class_id] + true_positives detections_total[class_id] = detections_total[class_id] + detections gt_instances_total[class_id] = gt_instances_total[class_id] + gt_instances print("---- " + class_name + " ----") print("gt_instances: ", gt_instances) print("true positives: ", true_positives) print("detections: ", detections) print("true_positives_total: ", true_positives_total[class_id]) print("detections_total: {}", detections_total[class_id]) #print("(Predicted) Scores:" , gt_scores_total, pred_scores_total) precisions_dict[image_id] = np.mean(precisions) recall_dict[image_id] = np.mean(recalls) # store APs.append(AP) #title = str(image_id) + 'no_mask' visualize.display_instances(image[..., :3], r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['class_scores'], r['scores'], figsize=(8,8), show_mask=False, savepath=image_id) # added scores $ #visualize.display_differences(image[..., :3], gt_bbox, gt_class_id, gt_mask, gt_score, r['rois'], r['class_ids'], r['class_scores'], # r['scores'], r['masks'], dataset_val.class_names,class_id, show_mask=False, savepath=image_id) #Analyze activations import tensorflow as tf activations = self.model.run_graph([image], [ ("input_image", tf.identity(self.model.keras_model.get_layer("input_image").output)), ("conv1", self.model.keras_model.get_layer("conv1").output), ("res2c_out", self.model.keras_model.get_layer("res2c_out").output), ("res3c_out", self.model.keras_model.get_layer("res3c_out").output), ("fpn_p4", self.model.keras_model.get_layer("fpn_p4").output), ("rpn_bbox", self.model.keras_model.get_layer("rpn_bbox").output), ("roi", self.model.keras_model.get_layer("ROI").output), ]) conv_sum = activations["conv1"][0,:,:,:].mean(axis=2) conv_sum=np.expand_dims(conv_sum, axis=2) res2c_sum = activations["res2c_out"][0,:,:,:].mean(axis=2) res2c_sum=np.expand_dims(res2c_sum, axis=2) print(activations["conv1"][0,:,:,:].shape) print(conv_sum.shape) #visualize.display_images(np.transpose(activations["input_image"][0,:,:,:4], [2, 0, 1]),titles=[image_id,2,3,4,5,6]) #visualize.display_images(np.transpose(activations["conv1"][0,:,:,:], [2, 0, 1]),titles=["Conv1", image_id, 3, 4,5,6,7,8,9,10]) #visualize.display_images(np.transpose(conv_sum, [2, 0, 1]),titles=["Conv1_Sum", image_id]) #visualize.display_images(np.transpose(res2c_sum, [2, 0, 1]),titles=["Res2C_Sum", image_id]) #visualize.display_images(np.transpose(activations["res3c_out"][0,:,:,:5], [2, 0, 1]),titles=["RES3C", image_id,3 ,4,5,6,7,8,9,10]) # calculate the mean AP, precision, recall, f1, MSE across all images mAP = np.mean(APs) for items in self.dataset.class_info: class_name = items["name"] if class_name != "Hen": continue class_id = items["id"] precision = true_positives_total[class_id]/detections_total[class_id] recall = true_positives_total[class_id]/gt_instances_total[class_id] MSE = np.mean(squared_errors_total[class_id]) print(class_name + " :") print("mAp: ", mAP) print("precision: ", precision) print("recall: ", recall) print("MSE: ", MSE) np.savetxt(f, gt_scores_total[class_id], delimiter=',') np.savetxt(f, pred_scores_total[class_id], delimiter=',') f.write('Precision: {0} Recall: {1} MSE: {2}\n'.format(precision, recall, MSE)) return 0
# If high confident skip if r['scores'][j] >= 0.8: continue box = r['rois'][j:j+1] class_id = r['class_ids'][j:j+1] score = r['scores'][j:j+1] mask = r['masks'][:,:,j:j+1] found = False # Check appear before for k in range(1, 6): if i - k < 0: break _, _r = preds[i-k] gt_match, _, _ = utils.compute_matches(box, class_id, mask, _r['rois'], _r['class_ids'], _r['scores'], _r['masks'], iou_threshold=0.1, score_threshold=0.0) if gt_match[0] != -1 and _r['scores'][int(gt_match[0])] > 0.7: found = True break # Check appear after for k in range(1, 6): if i + k >= len(preds): break _, _r = preds[i+k] gt_match, _, _ = utils.compute_matches(box, class_id, mask, _r['rois'], _r['class_ids'], _r['scores'], _r['masks'], iou_threshold=0.1, score_threshold=0.0) if gt_match[0] != -1 and _r['scores'][int(gt_match[0])] > 0.7: