def compute_batch_ap(dataset, image_ids, verbose=1): APs = [] mask_IoU = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) # Run object detection #results = model.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0)#gave only one mask results = model.detect([image], verbose=0) # Compute AP over range 0.5 to 0.95 r = results[0] ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) #print(r['scores']) #print(r['masks'].shape) APs.append(ap) if verbose: info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) #print("{:3} {} AP: {:.2f}".format( #meta["image_id"][0], meta["original_image_shape"][0], ap)) return APs #this outputs mAPs per image. May be usefull to keep it like that
def compute_batch_ap(dataset, image_ids, verbose=1): APs = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) # Run object detection results = model.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0) # Compute AP over range 0.5 to 0.95 r = results[0] ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) APs.append(ap) if verbose: info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) print("{:3} {} AP: {:.2f}".format( meta["image_id"][0], meta["original_image_shape"][0], ap)) return APs
def compute_batch_ap(model, dataset, image_ids, config, verbose=1): assert isinstance(model, model_lib.MaskRCNN) APs = [] buckets = [image_ids[i:i + config.BATCH_SIZE] for i in range(0, len(image_ids), config.BATCH_SIZE)] for images_id in buckets: if len(images_id) != config.BATCH_SIZE: continue images = [] images_meta = [] for image_id in images_id: # Load image log.debug('loading image %s' % image_id) image, image_meta, gt_class_id, gt_bbox, gt_mask = model_lib.load_image_gt(dataset, config, image_id, use_mini_mask=False) images.append(image) images_meta.append(image_meta) # Run object detection results = model.detect_molded(np.stack(images, axis=0), np.stack(images_meta, axis=0), verbose=0) assert config.BATCH_SIZE == len(results) # Compute AP over range 0.5 to 0.95 for r, image_id, image_meta in zip(results, images_id, images_meta): ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) APs.append(ap) if verbose: info = dataset.image_info[image_id] meta = model_lib.parse_image_meta(image_meta[np.newaxis, ...]) log.debug("{:3} {} AP: {:.2f}".format(meta["image_id"][0], meta["original_image_shape"][0], ap)) return APs
def evaluate_raspberry(model, dataset, annotations, eval_type="segm", limit=100, image_ids=None): # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. inference_config = RaspberryConfig() image_ids = np.random.choice(dataset_val.image_ids, limit) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP """AP, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'])""" AP = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) #print(str(APs)) print("mAP: ", np.mean(APs)) mAP=np.mean(APs) average.append(mAP) mAP=str(mAP) f.write("%s, " % mAP) print("evaluation completed and recorded")
def perform_eval(model, dataset): APs = [] for image_id in dataset.image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) # Run object detection results = model.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0) # Compute AP over range 0.5 to 0.95 r = results[0] if r['masks'].shape[-1] > 0: ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) APs.append(ap) info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) print("{:3} {} AP: {:.2f}".format( meta["image_id"][0], meta["original_image_shape"][0], ap)) print("Mean AP overa {} images: {:.4f}".format(len(APs), np.mean(APs)))
def compute_aps(detections, dataset, config, preds_cache={}, shape=256, thresh=0.95, just_stats=True): APs = [] n_rois = [] image_ids = [] n_bbox = [] for image_id, r in tqdm_notebook(detections.items()): image_ids.append(image_id) image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) mask = np.where(r['scores'] > thresh) rois = r['rois'][np.where(r['scores'] > thresh)].astype(int) if (len(rois) + len(gt_bbox)) == 0: ap = np.nan elif len(rois) == 0: ap = 0.0 # if we have target boxes but no predicted boxes, precision is 0 elif len(gt_bbox) == 0: ap = 0.0 else: masks = r['masks'][:, :, mask[0]] scores = r['scores'][mask] class_ids = np.ones(rois.shape[0]) ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, rois, class_ids, scores, masks, iou_thresholds=list( np.arange(0.4, .8, 0.05)), verbose=0) APs.append( # average_precision_image(rois, r['scores'], gt_bbox, shape=shape) ap # average_precision_image(rois, r['scores'], gt_bbox, shape=shape) ) n_rois.append(rois.shape[0]) n_bbox.append(gt_bbox.shape[0]) print("mAP {:.4f} ".format(np.nanmean(APs))) print("mean n rois {:.2f} ".format(np.mean(n_rois))) return pd.DataFrame({ 'ap': APs, 'n_roi': n_rois, 'n_bbox': n_bbox }, index=image_ids)
def compute_metrics(images, gt_boxes, gt_class_ids, gt_masks, results_list) -> List[Tuple]: mAPs = [] APs_75 = [] APs_5 = [] recalls_75 = [] recalls_5 = [] roundness_list = [] for image, gt_bbox, gt_class_id, gt_mask, results in zip( images, gt_boxes, gt_class_ids, gt_masks, results_list): # Compute metrics r = results AP_75, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], iou_threshold=0.75) AP_5, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], iou_threshold=0.5) mAP = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=False) recall_75, _ = utils.compute_recall(r['rois'], gt_bbox, iou=0.75) recall_5, _ = utils.compute_recall(r['rois'], gt_bbox, iou=0.5) # Roundness contours = get_contours(r) img_roundness = avg_roundness(contours) mAPs.append(mAP) APs_75.append(AP_75) APs_5.append(AP_5) recalls_75.append(recall_75) recalls_5.append(recall_5) roundness_list.append(img_roundness) names = [ '1. mAP@IoU Rng', '2. mAP@IoU=75', '3. mAP@IoU=50', '4. Recall @ IoU=75', '5. Recall @ IoU=50', '6. Roundness' ] values_list = [mAPs, APs_75, APs_5, recalls_75, recalls_5, roundness_list] avg_values = [np.mean(values) for values in values_list] return list(zip(names, avg_values))
def compute_batch_ap(model, model_config, dataset, limit, verbose=1): """ Validates the model on the dataset in the provided directory, and computes validation metric (mAP). """ # Validation dataset dataset_val = AlliumDataset() dataset_val.load_allium(dataset, "val") dataset_val.prepare() if limit: image_ids = dataset_val.image_ids[:limit] else: image_ids = dataset_val.image_ids print("Images: {}\nClasses: {}".format(len(dataset_val.image_ids), dataset_val.class_names)) # Compute mAP APs = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, config, image_id, use_mini_mask=False) # Run object detection results = model.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0) # Compute AP over range 0.5 to 0.95 r = results[0] ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) APs.append(ap) if verbose: # info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) print("{:3} {} AP: {:.2f}".format( meta["image_id"][0], meta["original_image_shape"][0], ap)) # Print the results mAP = np.mean(APs) print("Average precisions are: {}".format(APs)) print("Mean average precision is: {}".format(mAP)) return mAP
def run_eval(dataset, show_image=False): APs = [] for image_id in dataset.image_ids: image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, inference_config, image_id, use_mini_mask=False) results = model.detect([image], verbose=1) r = results[0] #import pdb; pdb.set_trace() """ AP, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) print (AP, "-", precisions, "-", recalls, "-", overlaps) """ _, ax = plt.subplots(1, figsize=(16, 16)) AP = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) #print (AP) APs.append(AP) visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], figsize=(8, 8), ax=ax) #save image .. filename = dataset.image_reference(image_id) plt.savefig(os.path.join(OUTPUT_FOLDER, filename + ".png")) if show_image: plt.show() return APs
def compute_ap_range(dataset, model, inference_config): image_ids = dataset.image_ids #np.random.choice(dataset.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image_name = dataset.image_info[image_id]['id'] print(image_name + " ap range results:") image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) print("mAP Range: ", np.mean(APs))
# Compute mAP #APs_05: IoU = 0.5 #APs_all: IoU from 0.5-0.95 with increments of 0.05 image_ids = np.random.choice(dataset_val.image_ids, 489) APs_05 = [] APs_all = [] for image_id in image_ids: # Load images and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP_05, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs_05.append(AP_05) AP_all = \ utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs_all.append(AP_all) print("mAP: ", np.mean(APs_05)) print("mAP: ", np.mean(APs_all))
APs = [] for image_id in image_ids: # Load image and ground truth data image, _, class_id, boxes, r_mask, g_mask, b_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) #molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0) results = model.detect([image], verbose=0) r = results[0] # Compute AP over range ap = utils.compute_ap_range(boxes, class_id, r_mask, g_mask, b_mask, r['rois'], r['class_ids'], r['scores'], r['r_masks'], r['g_masks'], r['b_masks'], verbose=0) #AP, precisions, recalls, overlaps =\ # utils.compute_ap(boxes, class_id, r_mask, g_mask, b_mask, # r['rois'], r['class_ids'], r['scores'], r['r_masks'], r['g_masks'], r['b_masks']) APs.append(ap) #print("mAP: ", np.mean(APs)) print("Mean AP over {} images: {:.4f}".format(len(APs), np.mean(APs)))
def evaluate_model(model, config): """ Evaluate the loaded model on the target dataset :param model: the architecture model, with loaded weights :param config: the configuration class for this dataset """ # Automatically discriminate the dataset according to the config file if isinstance(config, configurations.TabletopConfigInference): # Load the validation dataset dataset_val = datasets.TabletopDataset() elif isinstance(config, configurations.YCBVideoConfigInference): dataset_val = datasets.YCBVideoDataset() dataset_val.load_dataset(args.dataset, "val") dataset_val.prepare() # Compute COCO-Style mAP @ IoU=0.5-0.95 in 0.05 increments # Running on all images #image_ids = np.random.choice(dataset_val.image_ids, 200) image_ids = dataset_val.image_ids APs = [] AP50s = [] AP75s = [] image_batch_vector = [] image_batch_eval_data = [] img_batch_count = 0 import time t_inference = 0 class image_eval_data: def __init__(self, image_id, gt_class_id, gt_bbox, gt_mask): self.IMAGE_ID = image_id self.GT_CLASS_ID = gt_class_id self.GT_BBOX = gt_bbox self.GT_MASK = gt_mask self.DETECTION_RESULTS = None print("Evaluating model...") progbar = Progbar(target=len(image_ids)) for idx, image_id in enumerate(image_ids): # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset_val, config, image_id, use_mini_mask=False) # Compose a vector of images and data image_batch_vector.append(image) image_batch_eval_data.append( image_eval_data(image_id, gt_class_id, gt_bbox, gt_mask)) img_batch_count += 1 # If a batch is ready, go on to detection # The last few images in the dataset will not be used if batch size > 1 if img_batch_count < config.BATCH_SIZE: continue # Run object detection t_start = time.time() results = model.detect(image_batch_vector, verbose=0) t_inference += (time.time() - t_start) assert len(image_batch_eval_data) == len(results) for eval_data, detection_results in zip(image_batch_eval_data, results): eval_data.DETECTION_RESULTS = detection_results # Compute mAP at different IoU (as msCOCO mAP is computed) AP = utils.compute_ap_range( eval_data.GT_BBOX, eval_data.GT_CLASS_ID, eval_data.GT_MASK, eval_data.DETECTION_RESULTS["rois"], eval_data.DETECTION_RESULTS["class_ids"], eval_data.DETECTION_RESULTS["scores"], eval_data.DETECTION_RESULTS['masks'], verbose=0) AP50, _, _, _ = utils.compute_ap( eval_data.GT_BBOX, eval_data.GT_CLASS_ID, eval_data.GT_MASK, eval_data.DETECTION_RESULTS["rois"], eval_data.DETECTION_RESULTS["class_ids"], eval_data.DETECTION_RESULTS["scores"], eval_data.DETECTION_RESULTS['masks'], iou_threshold=0.5) AP75, _, _, _ = utils.compute_ap( eval_data.GT_BBOX, eval_data.GT_CLASS_ID, eval_data.GT_MASK, eval_data.DETECTION_RESULTS["rois"], eval_data.DETECTION_RESULTS["class_ids"], eval_data.DETECTION_RESULTS["scores"], eval_data.DETECTION_RESULTS['masks'], iou_threshold=0.75) APs.append(AP) AP50s.append(AP50) AP75s.append(AP75) # Reset the batch info image_batch_vector = [] image_batch_eval_data = [] img_batch_count = 0 progbar.update(idx + 1) print("\nmAP[0.5::0.05::0.95]: ", np.mean(APs)) print("mAP[0.5]: ", np.mean(AP50s)) print("mAP[0.75]: ", np.mean(AP75s)) print("Inference time for", len(image_ids), "images: ", t_inference, "s \tAverage FPS: ", len(image_ids) / t_inference) return APs
def compute_batch_metrics(dataset): APs50 = [] APs75 = [] APs_range = [] ARs50 = [] ARs75 = [] ARs_range = [] APs_range_small = [] APs_range_medium = [] APs_range_large = [] ARs_range_small = [] ARs_range_medium = [] ARs_range_large = [] '''for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)''' for image_id in dataset.image_ids: image = dataset.load_image(image_id) '''gt_bbox=[] gt_class_id=[] for dict in dataset.image_info[image_id]['annotations']: gt_bbox.append(dict['bbox']) gt_class_id.append(dict['category_id']) cv2.rectangle(image, (dict['bbox'][1], dict['bbox'][0]), (dict['bbox'][3], dict['bbox'][2]), (255, 0, 0), 2) gt_bbox = np.array(gt_bbox) print('gt_bbox:,', gt_bbox) #print('gt_bbox.shape:', gt_bbox.shape) gt_class_id = np.array(gt_class_id) #zerod = np.zeros(len(gt_class_id)) #gt_class_id = np.vstack((gt_class_id, zerod))''' gt_bbox, gt_class_id = dataset.load_mask(image_id) '''print('gt_bbox:,', gt_bbox) if len(gt_bbox[0]) == 4: for i in range(len(gt_bbox)): cv2.rectangle(image, (gt_bbox[i][1], gt_bbox[i][0]), (gt_bbox[i][3], gt_bbox[i][2]), (255, 0, 0), 2)''' gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) faces = faceCascade.detectMultiScale(gray, scaleFactor=1.03, minNeighbors=6, minSize=(24, 24)) # Draw a rectangle around the faces pred_bbox = [] pred_scores = [] pred_class_id = [] count = 0 for (x, y, w, h) in faces: #cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2) pred_bbox.append([y, x, y + h, x + w]) pred_scores.append(1.) pred_class_id.append(1) cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2) count += 1 if count == 0: pred_bbox.append([]) pred_scores.append(1.) pred_class_id.append(1) pred_bbox = np.array(pred_bbox) pred_scores = np.array(pred_scores) pred_class_id = np.array(pred_class_id) #zerod = np.zeros(len(pred_scores)) #pred_scores = np.vstack((pred_scores, zerod)) #pred_class_id = np.vstack((pred_class_id, zerod)) #print(pred_bbox) r = { 'rois': pred_bbox, 'class_ids': pred_class_id, 'scores': pred_scores } '''print("r['rois']:", r['rois']) img = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) cv2.imshow('Video', img) cv2.waitKey(0)''' # Run object detection #results = model.detect([image], verbose=0) # Compute AP #r = results[0] if pred_bbox.shape != (1, 0): AP50, precisions50, recalls50, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, r['rois'], r['class_ids'], r['scores'], config=config, iou_threshold=0.5) #print('AP50:', AP50) #print('pred_bbox.shape:', pred_bbox.shape) AP75, precisions75, recalls75, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, r['rois'], r['class_ids'], r['scores'], config=config, iou_threshold=0.75) AP_range =\ utils.compute_ap_range(gt_bbox, gt_class_id, r['rois'], r['class_ids'], r['scores'], config=config) AR50, positive_ids50 =\ utils.compute_recall(r['rois'], gt_bbox, iou=0.5) AR75, positive_ids75 =\ utils.compute_recall(r['rois'], gt_bbox, iou=0.75) AR_range =\ utils.compute_ar_range(r['rois'], gt_bbox) else: AP50 = 0. AP75 = 0. AP_range = 0. AR50 = 0. AR75 = 0. AR_range = 0. #AR50 = recalls50[-2] #AR75 = recalls75[-2] #AR_range = recalls_range[-2] APs50.append(AP50) APs75.append(AP75) APs_range.append(AP_range) ARs50.append(AR50) ARs75.append(AR75) ARs_range.append(AR_range) gt_ind_small, gt_ind_medium, gt_ind_large = find_bbox_indices_by_size( gt_bbox) gt_bbox_small = gt_bbox[gt_ind_small, :] gt_bbox_medium = gt_bbox[gt_ind_medium, :] gt_bbox_large = gt_bbox[gt_ind_large, :] gt_class_id_small = gt_class_id[gt_ind_small] gt_class_id_medium = gt_class_id[gt_ind_medium] gt_class_id_large = gt_class_id[gt_ind_large] if pred_bbox.shape != (1, 0): pred_ind_small, pred_ind_medium, pred_ind_large = find_bbox_indices_by_size( r['rois']) pred_bbox_small = r['rois'][pred_ind_small, :] pred_bbox_medium = r['rois'][pred_ind_medium, :] pred_bbox_large = r['rois'][pred_ind_large, :] pred_class_id_small = r['class_ids'][pred_ind_small] pred_class_id_medium = r['class_ids'][pred_ind_medium] pred_class_id_large = r['class_ids'][pred_ind_large] pred_score_small = r['scores'][pred_ind_small] pred_score_medium = r['scores'][pred_ind_medium] pred_score_large = r['scores'][pred_ind_large] if gt_bbox_small.size != 0: if pred_bbox.shape != (1, 0): AP_range_small =\ utils.compute_ap_range(gt_bbox_small, gt_class_id_small, pred_bbox_small, pred_class_id_small, pred_score_small, config=config) AR_range_small =\ utils.compute_ar_range(pred_bbox_small, gt_bbox_small) else: AP_range_small = 0. AR_range_small = 0. APs_range_small.append(AP_range_small) ARs_range_small.append(AR_range_small) if gt_bbox_medium.size != 0: if pred_bbox.shape != (1, 0): AP_range_medium =\ utils.compute_ap_range(gt_bbox_medium, gt_class_id_medium, pred_bbox_medium, pred_class_id_medium, pred_score_medium, config=config) AR_range_medium =\ utils.compute_ar_range(pred_bbox_medium, gt_bbox_medium) else: AP_range_medium = 0. AR_range_medium = 0. APs_range_medium.append(AP_range_medium) ARs_range_medium.append(AR_range_medium) if gt_bbox_large.size != 0: if pred_bbox.shape != (1, 0): AP_range_large =\ utils.compute_ap_range(gt_bbox_large, gt_class_id_large, pred_bbox_large, pred_class_id_large, pred_score_large, config=config) AR_range_large =\ utils.compute_ar_range(pred_bbox_large, gt_bbox_large) else: AP_range_large = 0. AR_range_large = 0. APs_range_large.append(AP_range_large) ARs_range_large.append(AR_range_large) return APs50, APs75, APs_range, ARs50, ARs75, ARs_range, APs_range_small, APs_range_medium,\ APs_range_large, ARs_range_small, ARs_range_medium, ARs_range_large
# Run object detection results = model.detect_molded(np.expand_dims(image, 0), np.expand_dims(image_meta, 0), verbose=1) # Display results r = results[0] log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) # Compute AP over range 0.5 to 0.95 and print it utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=1) visualize.display_differences(image, gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], dataset.class_names, ax=get_ax(), show_box=False,
def compute_batch_ap(dataset, image_ids, verbose=1): APs_general = [] APs_ring = [] APs_crack = [] # loop throgh all val dataset for image_id in image_ids: # Load image ground truths image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) print(image_id) # Run object detection results = model.detect([image], verbose=0) # Compute AP over range 0.5 to 0.95 r = results[0] ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) APs_general.append(ap) #print('APs_general', APs_general) # get AP values for ring and crack separately AP_loop = [] for i in [1, 2]: #print("LOOP START", i) if gt_mask[:, :, gt_class_id == i].shape[-1] > 0: ap = utils.compute_ap_range( gt_bbox[gt_class_id == i], gt_class_id[gt_class_id == i], gt_mask[:, :, gt_class_id == i], r['rois'][r['class_ids'] == i], r['class_ids'][r['class_ids'] == i], r['scores'][r['class_ids'] == i], r['masks'][:, :, r['class_ids'] == i], verbose=0) AP_loop.append(ap) #print(ap) else: ap = np.nan AP_loop.append(ap) #print(ap) #print('AP_loop', AP_loop) APs_ring.append(AP_loop[0]) APs_crack.append(AP_loop[1]) if verbose: info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) #print("{:3} {} AP: {:.2f}".format( #meta["image_id"][0], meta["original_image_shape"][0], ap)) mAP_general = np.nanmean(APs_general) mAP_ring = np.nanmean(APs_ring) mAP_crack = np.nanmean(APs_crack) return mAP_general, mAP_ring, mAP_crack #this outputs mAPs per image. May be usefull to keep it like that
def compute_batch_ap(dataset, image_ids, verbose=1): """ # Load validation dataset if you need to use this function. dataset = slum.slumDataset() dataset.load_slum(folder_path,fol) """ APs = [] IOUs = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) # Run object detection results = model.detect_molded(image[np.newaxis], image_meta[np.newaxis], verbose=0) # Compute AP over range 0.5 to 0.95 r = results[0] #merge_masks. gt_merge_mask = np.zeros((gt_mask.shape[:2])) for i in range(gt_mask.shape[2]): gt_merge_mask = np.logical_or(gt_merge_mask, gt_mask[:, :, i]) pred_merge_mask = np.zeros((r['masks'].shape[:2])) for i in range(r['masks'].shape[2]): pred_merge_mask = np.logical_or(pred_merge_mask, r['masks'][:, :, i]) pred_merge_mask = np.expand_dims(pred_merge_mask, 2) #print(pred_merge_mask.shape) pred_merge_mask, wind, scale, pad, crop = utils.resize_image( pred_merge_mask, 1024, 1024) #print(pred_merge_mask.shape,gt_merge_mask.shape) iou = jaccard_similarity_score(np.squeeze(pred_merge_mask), gt_merge_mask) #mAP at 50 print("mAP at 50") ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], np.arange(0.5, 1.0), verbose=0) #Make sure ap doesnt go above 1 ! if ap > 1.0: ap = 1.0 APs.append(ap) IOUs.append(iou) if verbose: info = dataset.image_info[image_id] meta = modellib.parse_image_meta(image_meta[np.newaxis, ...]) print("{:3} {} AP: {:.2f} Image_id: {}, IOU: {}".format( meta["image_id"][0], meta["original_image_shape"][0], ap, image_id, iou)) return APs, IOUs
def evaluation(model, dataset_dir, subset): # Create directory if not os.path.exists(RESULTS_DIR): os.makedirs(RESULTS_DIR) submit_dir = "submit_{:%Y%m%dT%H%M%S}".format(datetime.datetime.now()) submit_dir = os.path.join(RESULTS_DIR, submit_dir) os.makedirs(submit_dir) # Read dataset dataset_val = vehDetectionDataset() dataset_val.load_evaluation(dataset_dir, subset) dataset_val.prepare() numImages_eval = dataset_val.image_ids.size # Prepare for saving as matlab file images_eval = [] mAP_all = [] mAP_all_range = [] precisions_all = [] recalls_all = [] overlaps_all = [] for image_id in range(0,numImages_eval): print(image_id) source_id = dataset_val.image_info[image_id]["id"] print(source_id) # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset_val, config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP mAP, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) # Compute AP range (0.5:0.05:0.95) mAP_range = \ utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], iou_thresholds = None, verbose = 1) # Append results from image mAP_all.append(mAP) mAP_all_range.append(mAP_range) images_eval.append(source_id) precisions_all.append(precisions) recalls_all.append(recalls) overlaps_all.append(overlaps) # Save image with shape polygon around vehicles. Bbox and mask can be activated, s. below visualize.display_instances( image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, scores=None,# r['scores'], show_bbox=False, show_mask=False, colors=None, figsize=(19.20,10.80)) # can also add title="Predictions" plt.box(on=None) # plt.box(False) plt.savefig(os.path.join(submit_dir, dataset_val.image_info[image_id]["id"])) plt.close() # plt.clf() print("Evaluation Process with Mask-RCNN Done. Files saved to ", submit_dir) print("mAP: ", np.mean(mAP_all)) # FK: save to Matlab files saveMatFileName = submit_dir + "\evaluation_maskrcnn_output.mat" savemat(saveMatFileName, {"mAP_all": mAP_all, "mAP_all_range": mAP_all_range, "images_eval": images_eval, "precisions_all": precisions_all, "recalls_all": recalls_all, "overlaps_all": overlaps_all})
for image_id in dataset.image_ids: image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, config), 0) results = model.detect([image], verbose=0) r = results[0] index = [x for x in range(len(r['class_ids'])) if r['class_ids'][x] <= 4] AP_pascal, precision, recall, overlap = utils.compute_ap( gt_bbox, gt_class_id, gt_mask, r["rois"][index], r["class_ids"][index], r["scores"][index], r['masks'][..., index]) AP_coco = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r["rois"][index], r["class_ids"][index], r["scores"][index], r['masks'][..., index]) # if r["rois"].shape[0]: # # Precision-Recall curve # visualize.plot_precision_recall(AP, precision, recall) # # Grid of ground truth objects and their predictions # visualize.plot_overlaps(gt_class_id, r['class_ids'], r['scores'], overlap, dataset.class_names) mAP_pascal.append(AP_pascal) mAP_coco.append(AP_coco) mprecision.append(precision) mrecall.append(recall) moverlap.append(overlap) print("mAP PASCAL", np.mean(mAP_pascal) * 100, "%")
flow_step += 1 P2, P3, P4, P5, P6 = warp.predict([key_P2, key_P3, key_P4, key_P5, key_P6, flow]) inputs=[image_metas, P2, P3, P4, P5, P6] result = maskrcnn.detect_molded(inputs) # Compute AP if (image_id+1) % args.num_frames == 0 or args.fix: if np.sum(result["scores"]) == 0: print("{} Fasle".format(image_id)) continue AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, result["rois"], result["class_ids"], result["scores"], result['masks']) AP50s.append(AP) AP = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, result["rois"], result["class_ids"], result["scores"], result['masks'], verbose=0) APs.append(AP) print("step: {:4d}, AP50: {:.3f}, mAP: {:.3f}".format(image_id, np.mean(AP50s), np.mean(APs))) # Save if args.is_save: save_name = args.save_dir + 'mask' + str(image_id) + '.png' colors = np.array(label_colors)/255.0 pred_img = visualize.display_instances(image[:,:,3:], result['rois'], result['masks'], result['class_ids'], dataset.class_names, result['scores'], colors = colors, save_name=save_name) print("step: {:4d}, AP50: {:.3f}, mAP: {:.3f}".format(image_id, np.mean(AP50s), np.mean(APs))) print("segmentation steps:", seg_step, "flow steps:", flow_step) if __name__ == '__main__': main()
def process( self, args ): """ Run inference with trained model """ # load validation / test dataset test_ds = ShipDataset( (768,768) ) test_ds.load_info( os.path.join( args.data_path, 'test' ) ) test_ds.prepare() # Load weights print("Loading weights ", args.model_pathname) self._model.load_weights(args.model_pathname, by_name=True) for sample in range ( 10 ): # pick random image image_id = random.choice( test_ds.image_ids ) image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( test_ds, self._config, image_id, use_mini_mask=False ) # get image info info = test_ds.image_info[ image_id ] print("image ID: {}.{} ({}) {}".format( info["source"], info["id"], image_id, test_ds.image_reference(image_id) ) ) # run object detection results = self._model.detect([image], verbose=1) r = results[0] # compute AP over range 0.5 to 0.95 and print it utils.compute_ap_range( gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=1) # display results visualize.display_instances( image, r['rois'], r['masks'], r['class_ids'], test_ds.class_names, r['scores'], title="Predictions", show_bbox=False) # display actual vs predicted differences visualize.display_differences( image, gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], test_ds.class_names, title="Actual vs Predict Difference" ) return
10)) # this has to be the number of ground truth TPs_bbox.append(np.zeros(10)) FPs_bbox.append(np.zeros(10)) FNs_bbox.append( np.repeat(gt_mask.shape[-1], 10)) # this has to be the number of ground truth APlist.append(np.zeros(10)) mask_normal = np.zeros(shape=(imgheight, imgheight)) mask_normal = np.reshape(mask_normal, (1024, 1024, 1)) else: ###calculate all the stuff mask_normal = r['masks'] # for the combined mask at the end ap = utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks'], verbose=0) #print(r['scores']) #print(r['masks'].shape) mAP.append(ap) #compute mask IoU IoU_m = utils.compute_overlaps_masks(gt_mask, r['masks']) IoU_m = np.nan_to_num(np.mean(IoU_m)) #change nans to 0 mask_IoU.append(IoU_m) #compute bbox IoU IoU_bbox = utils.compute_overlaps(gt_bbox, r['rois']) IoU_bbox = np.nan_to_num(np.mean(IoU_bbox))