def test(model, config, limit=None): dataset_test = S3DDataset() dataset_test.load_s3d('test') dataset_test.prepare() print("Start inferencing on %d images..." % len(dataset_test.image_info)) APs = [] import tqdm for i in tqdm.tqdm(range(len(dataset_test.image_info))): if limit is not None: if i >= int(limit): continue image, meta, gt_class_ids, gt_bbox, gt_mask = modellib.load_image_gt( dataset_test, config, i) result = model.detect([image], verbose=0)[0] bbox = result['rois'] mask = result['masks'] class_ids = result['class_ids'] scores = result['scores'] second_class_ids = result['second_class_ids'] second_scores = result['second_scores'] probs = result['probs'][0] # print(class_ids) # print(second_class_ids) # print(scores) # print(second_scores) # print(bbox.shape, gt_bbox.shape) # print(mask.shape, gt_mask.shape) # print(class_ids.shape, gt_class_ids.shape) # print(scores.shape) # 基础的结果 basemAP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask) delta = 0.0 # 计入概率次高分类之后的结果 for i in range(len(class_ids)): ori = class_ids[i] class_ids[i] = second_class_ids[i] mAP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask) class_ids[i] = ori if mAP - basemAP > 0: delta += mAP - basemAP if mAP >= 0 and mAP <= 1: APs.append(basemAP + delta) print('%.3f' % np.mean(APs))
def evaluate_gdxray(model, dataset, eval_type="bbox", limit=0, image_ids=None): """Runs GDXRay evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ image_ids = image_ids or dataset.image_ids random.shuffle(image_ids) limit = int(limit) num_processed = 0 APs = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) # Run object detection results = model.detect([image], verbose=0) # Compute AP r = results[0] AP, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks']) print("Image", image_id, "AP:", AP) num_processed += 1 APs.append(AP) if limit and num_processed > limit: break return np.mean(APs)
def compute_mean_AP(model, config, dataset, n_images): """ Compute VOC-Style mAP @ IoU=0.5 """ image_ids = np.random.choice(dataset.image_ids, n_images) APs = [] for image_id in image_ids: # Load image and ground truth data result = modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) if len(result) == 5: image, image_meta, class_ids, gt_bbox, gt_mask = result else: image, image_meta, gt_bbox, gt_mask = result class_ids = gt_bbox[:, 4] gt_bbox = gt_bbox[:, :4] molded_images = np.expand_dims(modellib.mold_image(image, config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, class_ids, r["rois"], r["class_ids"], r["scores"]) APs.append(AP) return np.mean(APs)
def evaluate_model(dataset, model, cfg): APs = list() for image_id in dataset.image_ids: # load image, bounding boxes and masks for the image id image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset, cfg, image_id, use_mini_mask=False) # convert pixel values (e.g. center) scaled_image = modellib.mold_image(image, cfg) # convert image into one sample sample = expand_dims(scaled_image, 0) # make prediction yhat = model.detect(sample, verbose=0) # extract results for first sample r = yhat[0] try: # calculate statistics, including AP AP, _, _, _ = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) # store APs.append(AP) print("good image id : %d" % image_id) print("AP : %.3f" % AP) except: print("bad image id : %d" % image_id) # calculate the mean AP across all images mAP = mean(APs) return mAP
def basicResults(): basemAP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask) if basemAP >= 0 and basemAP <= 1: APs1.append(basemAP) fout.write('%d %f\n' % (i, basemAP))
def secondClassResults(): # 基础的结果 basemAP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask) delta = 0.0 # 计入概率次高分类之后的结果 for i in range(len(class_ids)): ori = class_ids[i] class_ids[i] = second_class_ids[i] mAP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask) class_ids[i] = ori if mAP - basemAP > 0: delta += mAP - basemAP if basemAP >= 0 and basemAP <= 1: APs2.append(basemAP + delta)
def compute_batch_ap(image_ids): APs = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset, config, image_id, use_mini_mask=False) # Run object detection results = model.detect([image], verbose=0) # Compute AP r = results[0] AP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, gt_class_id, r['rois'], r['class_ids'], r['scores']) APs.append(AP) return APs
def compute_map(iou_th): mean_aps_this_th = torch.zeros(len(dataloader_eval_covid), dtype=torch.float) for id, b in enumerate(dataloader_eval_covid): X, y = b if device == torch.device('cuda'): X, y['labels'], y['boxes'], y['masks'] = X.to( device), y['labels'].to(device), y['boxes'].to( device), y['masks'].to(device) images = [im for im in X] lab = {} # THIS IS IMPORTANT!!!!! # get rid of the first dimension (batch) # IF you have >1 images, make another loop # REPEAT: DO NOT USE BATCH DIMENSION lab['boxes'] = y['boxes'].squeeze_(0) lab['labels'] = y['labels'].squeeze_(0) lab['masks'] = y['masks'].squeeze_(0) image = [X.squeeze_(0)] # remove the batch dimension out = maskrcnn_model(image) # scores + bounding boxes + labels + masks scores = out[0]['scores'] bboxes = out[0]['boxes'] classes = out[0]['labels'] # remove the empty dimension, # output_size x 512 x 512 predict_mask = out[0]['masks'].squeeze_(1) > 0.5 if len(scores) > 0 and len(lab['labels']) > 0: # threshold for the masks: mAP, _, _, _ = utils.compute_ap(lab['boxes'], lab['labels'], lab['masks'], bboxes, classes, scores, predict_mask, iou_threshold=iou_th) mean_aps_this_th[id] = mAP elif not len(scores) and not len(lab['labels']): mean_aps_this_th[id] = 1 elif not len(scores) and len(lab['labels']) > 0: continue elif len(scores) > 0 and not len(y['labels']): continue print("AP @{0:.2f}:{1:.2f}".format(iou_th.item(), mean_aps_this_th.mean().item())) return mean_aps_this_th, mean_aps_this_th.mean().item()
def evaluate_wpif(model, config, dataset, eval_type="bbox", limit=0): """ Evaluation on WPIF dataset, using VOC-Style mAP # IoU=0.5 for bbox @TODO: add segment evaluation :param model: :param config: :param dataset: :param eval_type: :param limit: :return: """ image_ids = dataset.image_ids if limit: image_ids = np.random.choice(dataset.image_ids, limit) t_prediction = 0 t_start = time.time() APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, config), 0) # Run object detection t = time.time() results = model.detect([image], verbose=0) r = results[0] t_prediction += (time.time() - t) # Compute AP AP, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, r["rois"], r["class_ids"], r["scores"]) APs.append(AP) print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start) print("mAP: ", np.mean(APs))
def evaluate_apc(model, config, dataset, eval_type="bbox", limit=0): """ Evaluation on APC dataset, using VOC-Style mAP # IoU=0.5 for bbox @TODO: add segment evaluation :param model: :param config: :param dataset: :param eval_type: :param limit: :return: """ image_ids = dataset.image_ids if limit: image_ids = np.random.choice(dataset.image_ids, limit) t_prediction = 0 t_start = time.time() APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, config), 0) # Run object detection t = time.time() results = model.detect([image], verbose=0) r = results[0] t_prediction += (time.time() - t) # Compute AP AP, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, r["rois"], r["class_ids"], r["scores"]) APs.append(AP) print("Prediction time: {}. Average {}/image".format( t_prediction, t_prediction / len(image_ids))) print("Total time: ", time.time() - t_start) print("mAP: ", np.mean(APs))
def evaluate_maskrcnn(dataset_val,inference_config = InferenceConfig()): image_ids = np.random.choice(dataset_val.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) print("mAP: ", np.mean(APs)) return APs
def compute_map(model, iou_th, dl, device, mask_th): mean_aps_this_th = torch.zeros(len(dl), dtype=torch.float) for v, b in enumerate(dl): x, y = b if device == torch.device('cuda'): x, y['labels'], y['boxes'], y['masks'] = x.to( device), y['labels'].to(device), y['boxes'].to( device), y['masks'].to(device) lab = { 'boxes': y['boxes'].squeeze_(0), 'labels': y['labels'].squeeze_(0), 'masks': y['masks'].squeeze_(0) } image = [x.squeeze_(0)] # remove the batch dimension out = model(image) # scores + bounding boxes + labels + masks scores = out[0]['scores'] bboxes = out[0]['boxes'] classes = out[0]['labels'] # remove the empty dimension, # output_size x 512 x 512 predict_mask = out[0]['masks'].squeeze_(1) > mask_th if len(scores) > 0 and len(lab['labels']) > 0: # threshold for the masks: ap, _, _, _ = utils.compute_ap(lab['boxes'], lab['labels'], lab['masks'], bboxes, classes, scores, predict_mask, iou_threshold=iou_th) mean_aps_this_th[v] = ap elif not len(scores) and not len(lab['labels']): mean_aps_this_th[v] = 1 elif not len(scores) and len(lab['labels']) > 0: continue elif len(scores) > 0 and not len(y['labels']): continue return mean_aps_this_th.mean().item()
def evaluate_engine(model, dataset, inference_config, eval_type="bbox", limit=0, image_ids=None): """Runs official COCO evaluation. dataset: A Dataset object with valiadtion data eval_type: "bbox" or "segm" for bounding box or segmentation evaluation limit: if not 0, it's the number of images to use for evaluation """ # Pick images from the dataset image_ids = np.random.choice(dataset.image_ids, 100) # Limit to a subset if limit: image_ids = image_ids[:limit] t_prediction = 0 t_start = time.time() APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) print("mAP: ", np.mean(APs)) print("Total time: ", time.time() - t_start)
def evaluate(self): self.yolo.model = self.model # gather all detections and annotations all_detections = [[ None for i in range(self.generator.num_classes()) ] for j in range(self.generator.size())] all_annotations = [[ None for i in range(self.generator.num_classes()) ] for j in range(self.generator.size())] for i in range(self.generator.size()): raw_image = self.generator.load_image(i) raw_height, raw_width, raw_channels = raw_image.shape # make the boxes and the labels pred_boxes = self.yolo.predict(raw_image) score = np.array([box.score for box in pred_boxes]) pred_labels = np.array([box.label for box in pred_boxes]) if len(pred_boxes) > 0: pred_boxes = np.array([[ box.xmin * raw_width, box.ymin * raw_height, box.xmax * raw_width, box.ymax * raw_height, box.score ] for box in pred_boxes]) else: pred_boxes = np.array([[]]) # sort the boxes and the labels according to scores score_sort = np.argsort(-score) pred_labels = pred_labels[score_sort] pred_boxes = pred_boxes[score_sort] # copy detections to all_detections for label in range(self.generator.num_classes()): all_detections[i][label] = pred_boxes[pred_labels == label, :] annotations = self.generator.load_annotation(i) # copy detections to all_annotations for label in range(self.generator.num_classes()): all_annotations[i][label] = annotations[ annotations[:, 4] == label, :4].copy() # compute mAP by comparing all detections and all annotations average_precisions = {} for label in range(self.generator.num_classes()): false_positives = np.zeros((0, )) true_positives = np.zeros((0, )) scores = np.zeros((0, )) num_annotations = 0.0 for i in range(self.generator.size()): detections = all_detections[i][label] annotations = all_annotations[i][label] num_annotations += annotations.shape[0] detected_annotations = [] for d in detections: scores = np.append(scores, d[4]) if annotations.shape[0] == 0: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) continue overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations) assigned_annotation = np.argmax(overlaps, axis=1) max_overlap = overlaps[0, assigned_annotation] if max_overlap >= self.iou_threshold and assigned_annotation not in detected_annotations: false_positives = np.append(false_positives, 0) true_positives = np.append(true_positives, 1) detected_annotations.append(assigned_annotation) else: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) # no annotations -> AP for this class is 0 (is this correct?) if num_annotations == 0: average_precisions[label] = 0 continue # sort by score indices = np.argsort(-scores) false_positives = false_positives[indices] true_positives = true_positives[indices] # compute false positives and true positives false_positives = np.cumsum(false_positives) true_positives = np.cumsum(true_positives) # compute recall and precision recall = true_positives / num_annotations precision = true_positives / np.maximum( true_positives + false_positives, np.finfo(np.float64).eps) # compute average precision average_precision = compute_ap(recall, precision) average_precisions[label] = average_precision return average_precisions
results = model.detect([original_image], verbose=1) r = results[0] visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax=get_ax(), target=VIS_DIR+"result_" + str(image_id) + ".jpg") ########################################## ###### EVALUATION ###### ########################################## # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. image_ids = np.random.choice(dataset_val.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, r["rois"], r["class_ids"], r["scores"]) APs.append(AP) print("mAP: ", np.mean(APs))
def val(self): APs = [] for i, (images, targets) in enumerate(self.val_data_loader): if i == 10: break images = self.to_var(images) targets = self.to_var(targets) with torch.no_grad(): output = self.net(images) output = utils.non_max_suppression(output, 80, conf_thres=self.conf_thres, nms_thres=self.nms_thres) # Compute average precision for each sample for sample_i in range(targets.size(0)): correct = [] # Get labels for sample where width is not zero (dummies) annotations = targets[sample_i, targets[sample_i, :, 3] != 0] # Extract detections detections = output[sample_i] if detections is None: # If there are no detections but there are annotations mask as zero AP if annotations.size(0) != 0: APs.append(0) continue # Get detections sorted by decreasing confidence scores detections = detections[np.argsort(-detections[:, 4])] # If no annotations add number of detections as incorrect if annotations.size(0) == 0: correct.extend([0 for _ in range(len(detections))]) else: # Extract target boxes as (x1, y1, x2, y2) target_boxes = torch.FloatTensor(annotations[:, 1:].shape) target_boxes[:, 0] = (annotations[:, 1] - annotations[:, 3] / 2) target_boxes[:, 1] = (annotations[:, 2] - annotations[:, 4] / 2) target_boxes[:, 2] = (annotations[:, 1] + annotations[:, 3] / 2) target_boxes[:, 3] = (annotations[:, 2] + annotations[:, 4] / 2) target_boxes *= self.image_size detected = [] for *pred_bbox, conf, obj_conf, obj_pred in detections: pred_bbox = torch.FloatTensor(pred_bbox).view(1, -1) # Compute iou with target boxes iou = utils.bbox_iou(pred_bbox, target_boxes) # Extract index of largest overlap best_i = np.argmax(iou) # If overlap exceeds threshold and classification is correct mark as correct if iou[best_i] > self.iou_thres and obj_pred == annotations[ best_i, 0] and best_i not in detected: correct.append(1) detected.append(best_i) else: correct.append(0) # Extract true and false positives true_positives = np.array(correct) false_positives = 1 - true_positives # Compute cumulative false positives and true positives false_positives = np.cumsum(false_positives) true_positives = np.cumsum(true_positives) # Compute recall and precision at all ranks recall = true_positives / annotations.size( 0) if annotations.size(0) else true_positives precision = true_positives / np.maximum( true_positives + false_positives, np.finfo(np.float64).eps) # Compute average precision AP = utils.compute_ap(recall, precision) APs.append(AP) return np.mean(APs)
plt.imsave(str(img)+'.png', image) mask,gt_class_ids = test_dataset.load_mask(img) #print ('mask', mask.shape, image.shape) results = model.detect([image], verbose=1) r = results[0] #print (r.keys()) plt.imsave('gt' + str(img)+'.png', mask[:,:,0]) #print ("get class ids") #print (r['rois'], r['class_ids'], r['scores']) # for the base model-keep only cow predictions present_class = r['class_ids'] for idx, cls in enumerate(present_class): if cls not in good_classes: np.delete(r['rois'], idx, axis=0) np.delete(r['class_ids'], idx, axis=0) np.delete(r['scores'], idx, axis=0) #print ('class', r['rois'].shape, r['scores'].shape, r['class_ids'].shape, r['masks'].shape, gt_bbox.shape, mask.shape) AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_ids, mask, r['rois'], r['class_ids'], r['scores'], r['masks'], iou_threshold = th) print (AP, precisions, recalls) AP_list.append(AP) print (np.mean(AP_list), th) all_AP_list.append(np.mean(AP_list)) print ("mAP", np.mean(AP_list)) all_AP_list.append(np.mean(all_AP_list)) print ('all_AP_list', all_AP_list) #pickle.dump(str(all_AP_list), open("coco_cows_val_base_640.r","wb"))
original_image2, image_meta2, gt_class_id2, gt_bbox2, gt_mask2, gt_keypoint2 =\ modellib.load_image_gt_keypoints(val_dataset_keypoints, inference_config, 2*image_id+1, augment=False,use_mini_mask=inference_config.USE_MINI_MASK) if(inference_config.USE_MINI_MASK): gt_mask1 = utils.expand_mask(gt_bbox1,gt_mask1,original_image1.shape) gt_mask2 = utils.expand_mask(gt_bbox2,gt_mask2,original_image2.shape) results = model.detect_keypoint([original_image1, original_image2], verbose=0) r1 = results[0] r2 = results[1] try: AP.append(utils.compute_ap(gt_bbox1, gt_class_id1, gt_mask1, r1['rois'], r1['class_ids'], r1['scores'], r1['masks'])[0]) except ValueError: print('predict no person') AP.append(0) try: AP.append(utils.compute_ap(gt_bbox2, gt_class_id2, gt_mask2, r2['rois'], r2['class_ids'], r2['scores'], r2['masks'])[0]) except ValueError: print('predict no person') AP.append(0) try: OKS.append(OKS_algorithm(gt_bbox1, gt_keypoint1, r1)) except ValueError: print('predict no person') OKS.append(0)
# Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset_test, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] #IF PREDICT SOMETHING if not r['masks'].shape[0] == 0: # Compute AP AP, precisions, recalls, overlaps, pred_match, gt_match = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r["masks"], iou_threshold=0.25) ev_ids.append(r['class_ids']) ev_scores.append(r['scores']) ev_pred_match.append(pred_match) ev_gt_match.append(gt_match) # print((gt_match)) APs.append(AP) if image_id % 500 == 1 or image_id == len(image_ids): print(image_id) with open('evaluationVarsM/class_ids.pkl', 'wb') as f: pickle.dump(ev_ids, f) with open('evaluationVarsM/scores.pkl', 'wb') as f: pickle.dump(ev_scores, f)
iou_thresholds = np.arange(0.5,0.95,0.05) for i in iou_thresholds: APs_i = [] P_Kaggle_i = [] print('threshold: {}'.format(i)) for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precision_kaggle, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'], iou_threshold = i) APs_i.append(AP) P_Kaggle_i.append(precision_kaggle) APs.append(APs_i) P_Kaggles.append(P_Kaggle_i) print("mAP: ", np.mean(np.array(APs),axis=1)) print("Precision Kaggle: ", np.mean(np.array(P_Kaggles),axis=1)) ## def rle_encoding(mask): ''' x: numpy array of shape (height, width), 1 - mask, 0 - background Returns run length as list
for image_id in tqdm(dataset_val.image_ids): # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset_val, InferenceConfig, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, gt_class_id, r["rois"], r["class_ids"], r["scores"], iou_threshold=0.5) APs.append(AP) print("mAP: ", np.mean(APs)) elif args.command == "predict": class InferenceConfig(NucleiConfig): # Set batch size to 1 since we'll be running inference on # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU GPU_COUNT = 1 IMAGES_PER_GPU = 1 DETECTION_NMS_THRESHOLD = 0.3
dataset_val.class_names, r['scores'], ax=get_ax()) # ## Evaluation # In[14]: # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. image_ids = np.random.choice(dataset_val.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) print("mAP: ", np.mean(APs)) # In[ ]:
def evaluate(self, model, imgs, obj_threshold=0.3, nms_threshold=0.3, iou_threshold=0.5): """ # Arguments model : The model to evaluate. imgs : list of parsed test_img dictionaries. obj_threshold : The score confidence threshold to use for detections. nms_threshold : The threshold used to consider when a detection is positive or negative. # Returns A dict mapping class names to mAP scores. """ # gather all detections and annotations test_size = len(imgs) all_detections = [[None for i in range(self.n_class)] for j in range(test_size)] all_annotations = [[None for i in range(self.n_class)] for j in range(test_size)] ious = [] for i in range(test_size): image_name = imgs[i]['filename'] if '.jpg' not in image_name and '.png' not in image_name: image_name += '.jpg' raw_image = cv2.imread(image_name) raw_height, raw_width, raw_channels = raw_image.shape # make the boxes and the labels pred_boxes = self.predict(model, raw_image, obj_threshold, nms_threshold) score = np.array([box.score for box in pred_boxes]) pred_labels = np.array([box.label for box in pred_boxes]) if len(pred_boxes) > 0: pred_boxes = np.array([[ box.xmin * raw_width, box.ymin * raw_height, box.xmax * raw_width, box.ymax * raw_height, box.score ] for box in pred_boxes]) else: pred_boxes = np.array([[]]) # sort the boxes and the labels according to scores score_sort = np.argsort(-score) pred_labels = pred_labels[score_sort] pred_boxes = pred_boxes[score_sort] # copy detections to all_detections for label in range(self.n_class): all_detections[i][label] = pred_boxes[pred_labels == label, :] annotations = load_annotation(imgs, i, self.labels) # copy detections to all_annotations for label in range(self.n_class): all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy() # compute mAP by comparing all detections and all annotations average_precisions = {} for label in range(self.n_class): false_positives = np.zeros((0, )) true_positives = np.zeros((0, )) scores = np.zeros((0, )) num_annotations = 0.0 for i in range(test_size): detections = all_detections[i][label] annotations = all_annotations[i][label] num_annotations += annotations.shape[0] detected_annotations = [] for d in detections: scores = np.append(scores, d[4]) if annotations.shape[0] == 0: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) continue overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations) assigned_annotation = np.argmax(overlaps, axis=1) max_overlap = overlaps[0, assigned_annotation] ious.append(max_overlap) if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations: false_positives = np.append(false_positives, 0) true_positives = np.append(true_positives, 1) detected_annotations.append(assigned_annotation) else: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) # no annotations -> AP for this class is 0 (is this correct?) if num_annotations == 0: average_precisions[label] = 0 continue # sort by score indices = np.argsort(-scores) false_positives = false_positives[indices] true_positives = true_positives[indices] # compute false positives and true positives false_positives = np.cumsum(false_positives) true_positives = np.cumsum(true_positives) # compute recall and precision recall = true_positives / num_annotations precision = true_positives / np.maximum( true_positives + false_positives, np.finfo(np.float64).eps) # compute average precision average_precision = compute_ap(recall, precision) average_precisions[label] = average_precision map_dict = {} # print evaluation for label, average_precision in average_precisions.items(): map_dict[self.labels[label]] = average_precision print(self.labels[label], '{:.4f}'.format(average_precision)) print('mAP: {:.4f}'.format( sum(average_precisions.values()) / len(average_precisions))) print('average IOU: {:.4f}'.format(np.mean(ious))) average_map = sum( average_precisions.values()) / len(average_precisions) return [average_map, map_dict, np.mean(ious)]
def segment_inference(**kwargs): data_base_dir = kwargs['data_base_dir'] dataset_type = kwargs['dataset_type'] image_id = kwargs['image_id'] epochs = kwargs['epochs'] use_edgelist = kwargs['use_edgelist'] outputs_base_dir = 'outputs' vis_result_save_dir = os.path.join(outputs_base_dir, 'visual_result', dataset_type) trained_model_dir = os.path.join(outputs_base_dir, 'snapshot') edgelist_result_dir = os.path.join(outputs_base_dir, 'edgelist') model_path = os.path.join(trained_model_dir, 'mask_rcnn_sketchyscene_' + epochs + '.h5') os.makedirs(vis_result_save_dir, exist_ok=True) config = SketchInferConfig() config.display() # val/test dataset dataset_infer = SketchDataset(data_base_dir) dataset_infer.load_sketches(dataset_type) dataset_infer.prepare() # Recreate the model in inference mode model = modellib.MaskRCNN(mode="inference", config=config, model_dir='', log_dir='') # Load trained weights (fill in path to trained weights here) assert model_path != "", "Provide path to trained weights" print("Loading weights from ", model_path) model.load_weights(model_path, by_name=True) original_image, _, gt_class_id, gt_bbox, gt_mask, _ = \ modellib.load_image_gt(dataset_infer, config, image_id - 1, use_mini_mask=False) log("original_image", original_image) log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) gt_seg_path = os.path.join(vis_result_save_dir, str(image_id) + '_gt.png') visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id, dataset_infer.class_names, title='Ground-Truth', save_path=gt_seg_path, fix_color=True) ## inference results = model.detect([original_image], verbose=1) r = results[0] pred_boxes = r["rois"] # (nRoIs, (y1, x1, y2, x2)) pred_class_ids = r["class_ids"] # (nRoIs) pred_scores = r["scores"] # (nRoIs) pred_masks = r["masks"] # (768, 768, nRoIs) log("pred_boxes", pred_boxes) log("pred_masks", pred_masks) if config.IGNORE_BG: # Use original_image(768, 768, 3) {0, 255} to filter pred_masks pred_masks = np.transpose(pred_masks, (2, 0, 1)) # (nRoIs, 768, 768) bin_input = original_image[:, :, 0] == 255 pred_masks[:, bin_input[:, :]] = 0 # (nRoIs, 768, 768) pred_masks = np.transpose(pred_masks, (1, 2, 0)) # (768, 768, nRoIs) # refine pred_masks(768, 768, nRoIs) with edge-list if use_edgelist: refined_pred_masks = \ refine_mask_with_edgelist(image_id, dataset_type, data_base_dir, edgelist_result_dir, pred_masks.copy(), pred_boxes) # caculating AP iou_thresholds = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) APs = np.zeros([len(iou_thresholds)], dtype=np.float32) APs_edg = np.zeros([len(iou_thresholds)], dtype=np.float32) for i in range(len(iou_thresholds)): iouThr = iou_thresholds[i] AP, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, pred_boxes, pred_class_ids, pred_scores, pred_masks, iou_threshold=iouThr) APs[i] = AP if use_edgelist: AP_edg, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, pred_boxes, pred_class_ids, pred_scores, refined_pred_masks, iou_threshold=iouThr) APs_edg[i] = AP_edg mAP = np.mean(APs) mAP_edg = np.mean(APs_edg) print('APs', APs) print('mAP', mAP) print('APs_edg', APs_edg) print('mAP_edg', mAP_edg) # save visual results visual_seg_path = os.path.join(vis_result_save_dir, str(image_id) + '_seg.png') visualize.display_instances(original_image, pred_boxes, pred_masks, pred_class_ids, dataset_infer.class_names, pred_scores, title='Normal result', save_path=visual_seg_path, fix_color=True) if use_edgelist: visual_seg_edg_path = os.path.join(vis_result_save_dir, str(image_id) + '_seg_edgelist.png') visualize.display_instances(original_image, pred_boxes, refined_pred_masks, pred_class_ids, dataset_infer.class_names, pred_scores, title='Result with edgelist', save_path=visual_seg_edg_path, fix_color=True)
r = results[0] visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'], dataset.class_names, r['scores'], ax=ax, title="Predictions") log("gt_class_id", gt_class_id) log("gt_bbox", gt_bbox) log("gt_mask", gt_mask) # Draw precision-recall curve AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r['rois'], r['class_ids'], r['scores'], r['masks']) visualize.plot_precision_recall(AP, precisions, recalls) # Grid of ground truth objects and their predictions visualize.plot_overlaps(gt_class_id, r['class_ids'], r['scores'], overlaps, dataset.class_names) # Compute VOC-style Average Precision def compute_batch_ap(image_ids): APs = [] for image_id in image_ids: # Load image image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset, config,
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax=get_ax()) plt.show() # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. image_ids = np.random.choice(dataset_val.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox[:,:4], gt_bbox[:,4], r["rois"], r["class_ids"], r["scores"]) APs.append(AP) print("mAP: ", np.mean(APs))
r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax=get_ax()) # ## Evaluation # In[14]: # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. image_ids = np.random.choice(dataset_val.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_bbox, gt_mask = modellib.load_image_gt( dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps = utils.compute_ap( gt_bbox[:, :4], gt_bbox[:, 4], r["rois"], r["class_ids"], r["scores"]) APs.append(AP) print("mAP: ", np.mean(APs))
def validation(model, device, valid_dataloader, csv_log_file, tensor_type=torch.cuda.FloatTensor, num_classes=8): # Set model to evaluation mode model.train(False) model.eval() # Variables to store detections all_detections = [] all_annotations = [] for batch_i, (_, images, labels) in enumerate( tqdm.tqdm(valid_dataloader, desc="Detecting objects")): images = Variable(images.type(tensor_type)) with torch.no_grad(): outputs = model(images) outputs = non_max_suppression(outputs, num_classes, conf_thres=0.8, nms_thres=0.4) for output, annotations in zip(outputs, labels): all_detections.append([np.array([]) for _ in range(num_classes)]) if output is not None: # Get predicted boxes, confidence scores and labels pred_boxes = output[:, :5].cpu().numpy() scores = output[:, 4].cpu().numpy() pred_labels = output[:, -1].cpu().numpy() # Order by confidence sort_i = np.argsort(scores) pred_labels = pred_labels[sort_i] pred_boxes = pred_boxes[sort_i] for label in range(num_classes): all_detections[-1][label] = pred_boxes[pred_labels == label] all_annotations.append([np.array([]) for _ in range(num_classes)]) if any(annotations[:, -1] > 0): annotation_labels = annotations[annotations[:, -1] > 0, 0].numpy() _annotation_boxes = annotations[annotations[:, -1] > 0, 1:] # Reformat to x1, y1, x2, y2 and rescale to image dimensions annotation_boxes = np.empty_like(_annotation_boxes) annotation_boxes[:, 0] = _annotation_boxes[:, 0] - _annotation_boxes[:, 2] / 2 annotation_boxes[:, 1] = _annotation_boxes[:, 1] - _annotation_boxes[:, 3] / 2 annotation_boxes[:, 2] = _annotation_boxes[:, 0] + _annotation_boxes[:, 2] / 2 annotation_boxes[:, 3] = _annotation_boxes[:, 1] + _annotation_boxes[:, 3] / 2 annotation_boxes *= 416 for label in range(num_classes): all_annotations[-1][label] = annotation_boxes[ annotation_labels == label, :] # Clear variables from gpu, collect garbage and clear gpu cache memory del images, labels gc.collect() torch.cuda.empty_cache() average_precisions = {} for label in range(num_classes): true_positives = [] scores = [] num_annotations = 0 for i in tqdm.tqdm(range(len(all_annotations)), desc=f"Computing AP for class '{label}'"): detections = all_detections[i][label] annotations = all_annotations[i][label] num_annotations += annotations.shape[0] detected_annotations = [] for *bbox, score in detections: scores.append(score) if annotations.shape[0] == 0: true_positives.append(0) continue overlaps = bbox_iou_numpy(np.expand_dims(bbox, axis=0), annotations) assigned_annotation = np.argmax(overlaps, axis=1) max_overlap = overlaps[0, assigned_annotation] if max_overlap >= 0.5 and assigned_annotation not in detected_annotations: true_positives.append(1) detected_annotations.append(assigned_annotation) else: true_positives.append(0) # no annotations -> AP for this class is 0 if num_annotations == 0: average_precisions[label] = 0 continue true_positives = np.array(true_positives) false_positives = np.ones_like(true_positives) - true_positives # sort by score indices = np.argsort(-np.array(scores)) false_positives = false_positives[indices] true_positives = true_positives[indices] # compute false positives and true positives false_positives = np.cumsum(false_positives) true_positives = np.cumsum(true_positives) # compute recall and precision recall = true_positives / num_annotations precision = true_positives / np.maximum( true_positives + false_positives, np.finfo(np.float64).eps) # compute average precision average_precision = compute_ap(recall, precision) average_precisions[label] = average_precision ap_list = [] for c, ap in average_precisions.items(): print(f"+ Class '{c}' - AP: {ap}") # Write ap details to csv file ap_list.append(ap) mAP = np.mean(list(average_precisions.values())) ap_list.append(mAP) print(f"mAP: {mAP}") #writer = csv.writer(csv_log_file) csv_log_file.writerow(ap_list) return model, mAP
results = model.detect([original_image], verbose=1) # original_image = dataset_val.load_image(image_id) r = results[0] visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax=get_ax()) # Compute VOC-Style mAP @ IoU=0.5 # Running on 10 images. Increase for better accuracy. image_ids = np.random.choice(dataset_val.image_ids, 10) APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask = \ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP AP, precisions, recalls, overlaps = \ utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks']) APs.append(AP) print("mAP: ", np.mean(APs))
print(r['scores']) ''' val mAP ''' if 0: ths = np.linspace(0.5, 0.95, 10) image_ids = dataset_val.image_ids APs = [] for image_id in image_ids: # Load image and ground truth data image, image_meta, gt_class_id, gt_bbox, gt_mask =\ modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False) molded_images = np.expand_dims( modellib.mold_image(image, inference_config), 0) # Run object detection results = model.detect([image], verbose=0) r = results[0] # Compute AP under different iou threshold AP = [] for th in ths: AP_th, precisions, recalls, overlaps =\ utils.compute_ap(gt_bbox, gt_class_id, r["rois"], r["class_ids"], r["scores"], iou_threshold=th) AP.append(AP_th) APs.append(AP) print("mAP: ", np.mean(APs))
def evaluate(self, generator, iou_threshold=0.3, score_threshold=0.3, max_detections=100, save_path=None): """ Evaluate a given dataset using a given model. code originally from https://github.com/fizyr/keras-retinanet # Arguments generator : The generator that represents the dataset to evaluate. model : The model to evaluate. iou_threshold : The threshold used to consider when a detection is positive or negative. score_threshold : The score confidence threshold to use for detections. max_detections : The maximum number of detections to use per image. save_path : The path to save images with visualized detections to. # Returns A dict mapping class names to mAP scores. """ # gather all detections and annotations all_detections = [[None for i in range(generator.num_classes())] for j in range(generator.size())] all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())] for i in range(generator.size()): raw_image = generator.load_image(i) raw_height, raw_width, raw_channels = raw_image.shape # make the boxes and the labels pred_boxes = self.predict(raw_image) score = np.array([box.score for box in pred_boxes]) pred_labels = np.array([box.label for box in pred_boxes]) if len(pred_boxes) > 0: pred_boxes = np.array([[ box.xmin * raw_width, box.ymin * raw_height, box.xmax * raw_width, box.ymax * raw_height, box.score ] for box in pred_boxes]) else: pred_boxes = np.array([[]]) # sort the boxes and the labels according to scores score_sort = np.argsort(-score) pred_labels = pred_labels[score_sort] pred_boxes = pred_boxes[score_sort] # copy detections to all_detections for label in range(generator.num_classes()): all_detections[i][label] = pred_boxes[pred_labels == label, :] annotations = generator.load_annotation(i) # copy detections to all_annotations for label in range(generator.num_classes()): all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy() # compute mAP by comparing all detections and all annotations average_precisions = {} for label in range(generator.num_classes()): false_positives = np.zeros((0, )) true_positives = np.zeros((0, )) scores = np.zeros((0, )) num_annotations = 0.0 for i in range(generator.size()): detections = all_detections[i][label] annotations = all_annotations[i][label] num_annotations += annotations.shape[0] detected_annotations = [] for d in detections: scores = np.append(scores, d[4]) if annotations.shape[0] == 0: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) continue overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations) assigned_annotation = np.argmax(overlaps, axis=1) max_overlap = overlaps[0, assigned_annotation] if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations: false_positives = np.append(false_positives, 0) true_positives = np.append(true_positives, 1) detected_annotations.append(assigned_annotation) else: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) # no annotations -> AP for this class is 0 (is this correct?) if num_annotations == 0: average_precisions[label] = 0 continue # sort by score indices = np.argsort(-scores) false_positives = false_positives[indices] true_positives = true_positives[indices] # compute false positives and true positives false_positives = np.cumsum(false_positives) true_positives = np.cumsum(true_positives) # compute recall and precision recall = true_positives / num_annotations precision = true_positives / np.maximum( true_positives + false_positives, np.finfo(np.float64).eps) # compute average precision average_precision = compute_ap(recall, precision) average_precisions[label] = average_precision return average_precisions
def evaluate(self, generator, iou_threshold=0.3, score_threshold=0.3, max_detections=100, save_path=None): """ Evaluate a given dataset using a given model. code originally from https://github.com/fizyr/keras-retinanet # Arguments generator : The generator that represents the dataset to evaluate. model : The model to evaluate. iou_threshold : The threshold used to consider when a detection is positive or negative. score_threshold : The score confidence threshold to use for detections. max_detections : The maximum number of detections to use per image. save_path : The path to save images with visualized detections to. # Returns A dict mapping class names to mAP scores. """ # gather all detections and annotations all_detections = [[None for i in range(generator.num_classes())] for j in range(generator.size())] all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())] for i in range(generator.size()): raw_image = generator.load_image(i) raw_height, raw_width, raw_channels = raw_image.shape # make the boxes and the labels pred_boxes = self.predict(raw_image) score = np.array([box.score for box in pred_boxes]) pred_labels = np.array([box.label for box in pred_boxes]) if len(pred_boxes) > 0: pred_boxes = np.array([[box.xmin*raw_width, box.ymin*raw_height, box.xmax*raw_width, box.ymax*raw_height, box.score] for box in pred_boxes]) else: pred_boxes = np.array([[]]) # sort the boxes and the labels according to scores score_sort = np.argsort(-score) pred_labels = pred_labels[score_sort] pred_boxes = pred_boxes[score_sort] # copy detections to all_detections for label in range(generator.num_classes()): all_detections[i][label] = pred_boxes[pred_labels == label, :] annotations = generator.load_annotation(i) # copy detections to all_annotations for label in range(generator.num_classes()): all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy() # compute mAP by comparing all detections and all annotations average_precisions = {} for label in range(generator.num_classes()): false_positives = np.zeros((0,)) true_positives = np.zeros((0,)) scores = np.zeros((0,)) num_annotations = 0.0 for i in range(generator.size()): detections = all_detections[i][label] annotations = all_annotations[i][label] num_annotations += annotations.shape[0] detected_annotations = [] for d in detections: scores = np.append(scores, d[4]) if annotations.shape[0] == 0: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) continue overlaps = compute_overlap(np.expand_dims(d, axis=0), annotations) assigned_annotation = np.argmax(overlaps, axis=1) max_overlap = overlaps[0, assigned_annotation] if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations: false_positives = np.append(false_positives, 0) true_positives = np.append(true_positives, 1) detected_annotations.append(assigned_annotation) else: false_positives = np.append(false_positives, 1) true_positives = np.append(true_positives, 0) # no annotations -> AP for this class is 0 (is this correct?) if num_annotations == 0: average_precisions[label] = 0 continue # sort by score indices = np.argsort(-scores) false_positives = false_positives[indices] true_positives = true_positives[indices] # compute false positives and true positives false_positives = np.cumsum(false_positives) true_positives = np.cumsum(true_positives) # compute recall and precision recall = true_positives / num_annotations precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps) # compute average precision average_precision = compute_ap(recall, precision) average_precisions[label] = average_precision return average_precisions