def calculate_metrics(mask1, mask2): true_positives = metric.obj_tpr(mask1, mask2) false_positives = metric.obj_fpr(mask1, mask2) dc = metric.dc(mask1, mask2) hd = metric.hd(mask1, mask2) precision = metric.precision(mask1, mask2) recall = metric.recall(mask1, mask2) ravd = metric.ravd(mask1, mask2) assd = metric.assd(mask1, mask2) asd = metric.asd(mask1, mask2) return true_positives, false_positives, dc, hd, precision, recall, ravd, assd, asd
def avg_surface_distance_symmetric(test=None, reference=None, confusion_matrix=None, nan_for_nonexisting=True, voxel_spacing=None, connectivity=1, **kwargs): if confusion_matrix is None: confusion_matrix = ConfusionMatrix(test, reference) test_empty, test_full, reference_empty, reference_full = confusion_matrix.get_existence() if test_empty or test_full or reference_empty or reference_full: if nan_for_nonexisting: return float("NaN") else: return 0 test, reference = confusion_matrix.test, confusion_matrix.reference return metric.assd(test, reference, voxel_spacing, connectivity)
def total_score(pred_path,gt_path, file_names, metric_name): print(pred_path) gt_to_pred = {k:k for k in [0,1]} list_labels = sorted(gt_to_pred.keys()) score = dict() thre = 400 score['names'] = [] score['lesion'] = [] for name in file_names: ground_truth = gt_path.format(name) ground_truth = os.path.expanduser(ground_truth) image_gt = nibabel.load(ground_truth) image_gt= nibabel.funcs.as_closest_canonical(image_gt).get_data() image_gt = image_gt.reshape(image_gt.shape[:3]) pred = pred_path.format(name) pred = os.path.expanduser(pred) image_pred = nibabel.load(pred) affine = image_pred.affine voxel = [affine[0,0],affine[1,1],affine[2,2]] image_pred = image_pred.get_data() image_pred = image_pred.reshape(image_pred.shape[:3]) score['names'].append(name) if metric_name=='assd': score['lesion'].append(metric.assd(image_gt,image_pred,voxel)) elif metric_name=='rve': score['lesion'].append(metric.ravd(image_gt,image_pred)) else: score['lesion'].append(metric.dc(image_gt,image_pred)) print('Sample size: {}'.format(len(list(score.values())[0]))) for label in score.keys(): if label != 'names': print('Label: {}, {} mean: {}'.format(label, metric_name, round(np.mean(score[label]),2))) print('Label: {}, {} std: {}'.format(label, metric_name, round(np.std(score[label]),2))) return score
def get_scores(pred, label, vxlspacing): volscores = {} volscores['dice'] = metric.dc(pred, label) volscores['jaccard'] = metric.binary.jc(pred, label) volscores['voe'] = 1. - volscores['jaccard'] volscores['rvd'] = metric.ravd(label, pred) if np.count_nonzero(pred) == 0 or np.count_nonzero(label) == 0: volscores['assd'] = 0 volscores['msd'] = 0 else: # evalsurf = Surface(pred,label,physical_voxel_spacing = vxlspacing,mask_offset = [0.,0.,0.], reference_offset = [0.,0.,0.]) # volscores['assd'] = evalsurf.get_average_symmetric_surface_distance() volscores['assd'] = metric.assd(label, pred, voxelspacing=vxlspacing) volscores['msd'] = metric.hd(label, pred, voxelspacing=vxlspacing) return volscores
def calculate_validation_metrics(probas_pred, image_gt, class_labels=None, num_classes=5): classes = np.arange(probas_pred.shape[-1]) # determine valid classes (those that actually appear in image_gt). Some images may miss some classes classes = [c for c in classes if np.sum(image_gt == c) != 0] image_pred = probas_pred.argmax(-1) assert image_gt.shape == image_pred.shape accuracy = np.sum(image_gt == image_pred) / float(image_pred.size) class_metrics = {} y_true = convert_seg_flat_to_binary_label_indicator_array( image_gt.ravel(), num_classes).astype(int)[:, classes] y_pred = probas_pred.transpose(3, 0, 1, 2).reshape(num_classes, -1).transpose(1, 0)[:, classes] scores = roc_auc_score(y_true, y_pred, None) for i, c in enumerate(classes): true_positives = metric.obj_tpr(image_gt == c, image_pred == c) false_positives = metric.obj_fpr(image_gt == c, image_pred == c) dc = metric.dc(image_gt == c, image_pred == c) hd = metric.hd(image_gt == c, image_pred == c) precision = metric.precision(image_gt == c, image_pred == c) recall = metric.recall(image_gt == c, image_pred == c) ravd = metric.ravd(image_gt == c, image_pred == c) assd = metric.assd(image_gt == c, image_pred == c) asd = metric.asd(image_gt == c, image_pred == c) label = c if class_labels is not None and c in class_labels.keys(): label = class_labels[c] class_metrics[label] = { 'true_positives': true_positives, 'false_positives': false_positives, 'DICE\t\t': dc, 'Hausdorff dist': hd, 'precision\t': precision, 'recall\t\t': recall, 'rel abs vol diff': ravd, 'avg surf dist symm': assd, 'avg surf dist\t': asd, 'roc_auc\t\t': scores[i] } return accuracy, class_metrics
def compute_typical_metrics(seg_gt, seg_pred, labels): assert seg_gt.shape == seg_pred.shape mask_pred = np.zeros(seg_pred.shape, dtype=bool) mask_gt = np.zeros(seg_pred.shape, dtype=bool) for l in labels: mask_gt[seg_gt == l] = True mask_pred[seg_pred == l] = True vol_gt = np.sum(mask_gt) vol_pred = np.sum(mask_pred) try: cm = confusion_matrix( mask_pred.astype(int).ravel(), mask_gt.astype(int).ravel()) TN = cm[0][0] FN = cm[0][1] FP = cm[1][0] TP = cm[1][1] precision = TP / float(TP + FP) recall = TP / float(TP + FN) fpr = FP / float(FP + TN) false_omission_rate = FN / float(FN + TN) except: precision = np.nan recall = np.nan fpr = np.nan false_omission_rate = np.nan try: dice = metric.dc(mask_pred, mask_gt) if np.sum(mask_gt) == 0: dice = np.nan except: dice = np.nan try: assd = metric.assd(mask_gt, mask_pred) except: assd = np.nan return precision, recall, fpr, false_omission_rate, dice, assd, vol_gt, vol_pred
def main(FLAGS): # set GPU device to use os.environ['CUDA_VISIBLE_DEVICES'] = '3' # get the models to evaluate ckpts = glob(os.path.join(FLAGS.ckpt_dir, '*.h5')) # get data files and sort them image_files = os.listdir(FLAGS.test_X_dir) anno_files = os.listdir(FLAGS.test_y_dir) image_files.sort() anno_files.sort() for ckpt in ckpts: K.clear_session() # set model to load ckpt_name = os.path.basename(ckpt) # create a location to store evaluation metrics metrics = np.zeros((len(image_files), FLAGS.classes, 8)) overall_accuracy = np.zeros((len(image_files), )) # create a file writer to store the metrics excel_name = os.path.splitext(os.path.basename(ckpt))[0] + '.xlsx' writer = pd.ExcelWriter(excel_name) model = load_model(ckpt) for i in range(len(image_files)): # define path to the test data test_path = os.path.join(FLAGS.test_X_dir, image_files[i]) generator = FCN2DDatasetGenerator( test_path, batch_size=FLAGS.batch_size, subset='test', normalization=FLAGS.normalization, categorical_labels=True, num_classes=FLAGS.classes) # check if the images and annotations are the correct files print(image_files[i], anno_files[i]) preds = model.predict_generator(generator.generate(), steps=len(generator)) stamp = datetime.datetime.fromtimestamp( time.time()).strftime('date_%Y_%m_%d_time_%H_%M_%S') write_hdf5('fcn_predictions_' + stamp + '.h5', preds) pred_file = glob(os.path.join(FLAGS.predictions_temp_dir, '*.h5'))[0] pt_name = image_files[i].split('.')[0] new_name_raw = pt_name + ckpt_name new_file_raw = os.path.join(FLAGS.predictions_final_dir, new_name_raw) os.rename(pred_file, new_file_raw) ref = read_hdf5_multientry( os.path.join(FLAGS.test_y_dir, anno_files[i])) ref = np.squeeze(np.asarray(ref)) preds = read_hdf5(new_file_raw) preds = np.argmax(preds, axis=-1) overall_accuracy[i] = skm.accuracy_score(ref.flatten(), preds.flatten()) for j in range(FLAGS.classes): organ_pred = (preds == j).astype(np.int64) organ_ref = (ref == j).astype(np.int64) if np.sum(organ_pred) == 0 or np.sum(organ_ref) == 0: metrics[i, j, 0] = 0. metrics[i, j, 1] = 0. metrics[i, j, 2] = 1. metrics[i, j, 3] = 0. metrics[i, j, 4] = 0. metrics[i, j, 5] = 0. metrics[i, j, 6] = np.inf metrics[i, j, 7] = np.inf else: metrics[i, j, 0] = jaccard_index(organ_ref, organ_pred) metrics[i, j, 1] = dice_similarity_coefficient( organ_ref, organ_pred) metrics[i, j, 2] = relative_volume_difference( organ_ref, organ_pred) metrics[i, j, 3] = precision(organ_ref, organ_pred) metrics[i, j, 4] = recall(organ_ref, organ_pred) metrics[i, j, 5] = matthews_correlation_coefficient( organ_ref, organ_pred) metrics[i, j, 6] = mpm.hd95(organ_pred, organ_ref) metrics[i, j, 7] = mpm.assd(organ_pred, organ_ref) print(overall_accuracy[i]) print(metrics[i]) for k in range(metrics.shape[-1]): data = pd.DataFrame(metrics[:, :, k], columns=['bg', 'met']) data.to_excel(writer, sheet_name=str(k)) acc = pd.DataFrame(overall_accuracy, columns=['acc']) acc.to_excel(writer, sheet_name='acc') writer.save()
def wassd(x): try: val = assd(*x) except RuntimeError: val = numpy.inf return val
def calculate_binary_hd(y_true, y_pred, thres=0.5, spacing=[1, 1, 1]): y_true = np.squeeze(y_true) y_pred = np.squeeze(y_pred) y_true = np.where(y_true > thres, 1, 0) y_pred = np.where(y_pred > thres, 1, 0) return assd(y_pred, y_true, spacing)
def process(self, inputs, outputs): for input, output in zip(inputs, outputs): self.count += len(output["instances"]) gt_segm = self.annotations[input["file_name"]]["segm_per_region"] try: _ = output["instances"].pred_masks except AttributeError: continue pred_segm = downsample_points(output) doc_ahd = {cat: [] for cat in categories_list} doc_hd = {cat: [] for cat in categories_list} doc_hd95 = {cat: [] for cat in categories_list} doc_iou = {cat: [] for cat in categories_list} doc_acc = {cat: [] for cat in categories_list} for reg_type in range(len(categories_list)): gt, pred = gt_segm[reg_type], pred_segm[reg_type] # Both have points if len(gt) and len(pred): """"Peformed using medpy""" gt_mask = np.zeros((len(gt), input["height"], input["width"]), dtype=np.int8) for i in range(len(gt)): cv2.fillPoly(gt_mask[i], np.array([gt[i]]).astype(np.int32), 1) pred_mask = np.zeros((len(pred), input["height"], input["width"]), dtype=np.int8) for i in range(len(pred)): cv2.fillPoly(pred_mask[i], np.array([pred[i]]).astype(np.int32), 1) gt_mask = gt_mask.astype(np.uint8) gt_mask = (gt_mask * 255).astype(np.uint8) pred_mask = pred_mask.astype(np.uint8) pred_mask = (pred_mask * 255).astype(np.uint8) def compute_iou_and_accuracy(arrs, edge_mask1): intersection = cv2.bitwise_and(arrs, edge_mask1) union = cv2.bitwise_or(arrs, edge_mask1) intersection_sum = np.sum(intersection) union_sum = np.sum(union) iou = (intersection_sum) / (union_sum) total = np.sum(arrs) correct_predictions = intersection_sum accuracy = correct_predictions / total # print(iou, accuracy) return iou, accuracy iou_hd_dict = np.empty((len(gt), len(pred), 3)) # IOU, HD95, AHD for i, each_gt_instance in enumerate(gt_mask): for j, each_pred_instance in enumerate(pred_mask): inst_iou, _ = compute_iou_and_accuracy(gt_mask[i], pred_mask[j]) inst_hd95 = hd95(gt_mask[i], pred_mask[j]) inst_ahd = assd(gt_mask[i], pred_mask[j]) iou_hd_dict[i][j][0] = inst_iou iou_hd_dict[i][j][1] = inst_hd95 iou_hd_dict[i][j][2] = inst_ahd corr_matrix = np.empty((len(gt), 4)) corr_matrix[:, 0] = np.argmax(iou_hd_dict[:, :, 0], 1) for i, each_gt_instance_metric in enumerate(iou_hd_dict): corr_matrix[i, 1:] = each_gt_instance_metric[corr_matrix[i, 0].astype(np.int)] gt_mask_copy = gt_mask.copy() gt_mask_copy = np.repeat(gt_mask_copy[:, :, :, np.newaxis], 3, axis=3) def bbox2(img): rows = np.any(img, axis=1) cols = np.any(img, axis=0) rmin, rmax = np.where(rows)[0][[0, -1]] cmin, cmax = np.where(cols)[0][[0, -1]] return rmin, rmax, cmin, cmax for i, each_gt_instance in enumerate(gt_mask): rmin, rmax, cmin, cmax = bbox2(each_gt_instance) pos = (int(cmin), int((rmin + rmax) / 2)) gt_mask_copy[i] = cv2.cvtColor(each_gt_instance, cv2.COLOR_GRAY2RGB) cv2.putText(gt_mask_copy[i], f"{i}", pos, cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 0, 0), 2) gt_image = gt_mask_copy.sum(axis=0).clip(0, 255).astype(np.uint8) # x = plt.imshow(gt_image) # plt.show() pred_image = pred_mask.sum(axis=0).clip(0, 255) pred_image_copy = pred_image.copy() pred_image_copy = np.repeat(pred_image_copy[:, :, np.newaxis], 3, axis=2).astype(np.uint8) for i, each_pred_instance in enumerate(pred_mask): rmin, rmax, cmin, cmax = bbox2(each_pred_instance) pos = (int(cmin), int((rmin + rmax) / 2)) # pred_image_copy[i] = cv2.cvtColor(each_pred_instance, cv2.COLOR_GRAY2RGB) if i in corr_matrix[:, 0].astype(np.int): if self.write_to_csv: metrics = (corr_matrix[np.where(i == corr_matrix[:, 0]), 1:]).squeeze() if metrics.ndim > 1: metrics = metrics[np.argmax(metrics[:, 0])] gt_idx = np.where((metrics == corr_matrix[:, 1:]).all(axis=1))[0].item() self.metrics_for_csv.append( { "region_name": f"{osp.splitext('_'.join(input['file_name'].split('/')[-3:]))[0]}_gt_{reg_type}_{gt_idx}", "iou": metrics[0].round(2), "ahd": metrics[2].round(2), "hd95": metrics[1].round(2), } ) continue text_metrics = np.array2string( (corr_matrix[np.where(i == corr_matrix[:, 0]), 1:].round(4)).squeeze() ) cv2.putText( pred_image_copy, f"{i} - {text_metrics}", pos, cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 0, 0), 2, ) else: cv2.putText(pred_image_copy, f"{i}", pos, cv2.FONT_HERSHEY_SIMPLEX, 0.4, (0, 255, 0), 1) # x = plt.imshow(pred_image_copy) # plt.show() if not self.write_to_csv: save_path = "final_outputs/comparision/bmrcnn_masks_whew/" import os file_name = os.path.splitext("_".join(input["file_name"].split("/")[-3:]))[0] cv2.imwrite( save_path + f"{file_name}_gt_{reg_type}.jpg", cv2.cvtColor(gt_image, cv2.COLOR_RGB2BGR) ) cv2.imwrite( save_path + f"{file_name}_pred_{reg_type}.jpg", cv2.cvtColor(pred_image_copy, cv2.COLOR_RGB2BGR), ) # res_iou, res_accuracy = compute_iou_and_accuracy(pred_mask, gt_mask) # res_ahd, res_hd, res_hd95 = assd(pred_mask, gt_mask), hd(pred_mask, gt_mask), hd95(pred_mask, gt_mask) # self.ahd[categories_list[reg_type]].append(res_ahd) # self.hd[categories_list[reg_type]].append(res_hd) # self.hd95[categories_list[reg_type]].append(res_hd95) # self.hd[categories_list[reg_type]].append(hd(pred_mask, gt_mask)) # self.iou[categories_list[reg_type]].append(res_iou) # self.acc[categories_list[reg_type]].append(res_accuracy) # # doc_ahd[categories_list[reg_type]].append(res_ahd) # doc_hd[categories_list[reg_type]].append(res_hd) # doc_hd95[categories_list[reg_type]].append(res_hd95) # doc_hd[categories_list[reg_type]].append(hd(pred_mask, gt_mask)) # doc_iou[categories_list[reg_type]].append(res_iou) # doc_acc[categories_list[reg_type]].append(res_accuracy) # One has points # elif len(gt) ^ len(pred): # total_area = 0 # for each_pred in pred: # total_area += PolyArea(each_pred[:, 0], each_pred[:, 1]) # hd = total_area / 100 # self.ahd[categories_list[reg_type]].append(hd) # self.hd[categories_list[reg_type]].append(hd) # self.hd95[categories_list[reg_type]].append(hd) # self.iou[categories_list[reg_type]].append(0) # self.acc[categories_list[reg_type]].append(0) # Both Empty # elif len(gt) == 0 and len(pred) != 0: else: # self.hd[categories_list[reg_type]].append(0) pass # print("Over for doc") total_ahd = list() for l in doc_ahd.values(): total_ahd.extend(l) doc_ahd = np.mean(total_ahd) total_hd = list() for l in doc_hd.values(): total_hd.extend(l) doc_hd = np.mean(total_hd) total_hd95 = list() for l in doc_hd95.values(): total_hd95.extend(l) doc_hd95 = np.mean(total_hd95) total_iou = list() for l in doc_iou.values(): total_iou.extend(l) doc_iou = np.mean(total_iou) total_acc = list() for l in doc_acc.values(): total_acc.extend(l) doc_acc = np.mean(total_acc) self.doc_wise[input["file_name"]] = { "AHD": doc_ahd, "IOU": doc_iou, "HD": doc_hd, "HD95": doc_hd95, "ACC": doc_acc, }
def average_surface_distance(data1, data2, voxelspacing=None): data1, data2 = to_numpy(data1), to_numpy(data2) return assd(data1, data2, voxelspacing)
def process(self, inputs, outputs): for input, output in zip(inputs, outputs): self.count += len(output['instances']) gt_segm = self.annotations[input['file_name']]['segm_per_region'] try: _ = output['instances'].pred_masks except AttributeError: continue pred_segm = downsample_points(output) doc_ahd = {cat: [] for cat in categories_list} doc_hd = {cat: [] for cat in categories_list} doc_hd95 = {cat: [] for cat in categories_list} doc_iou = {cat: [] for cat in categories_list} doc_acc = {cat: [] for cat in categories_list} for reg_type in range(len(categories_list)): gt, pred = gt_segm[reg_type], pred_segm[reg_type] # Both have points if len(gt) and len(pred): gt_mask = np.zeros((input['height'], input['width']), dtype=np.int8) for i in gt: cv2.fillPoly(gt_mask, np.array([i]).astype(np.int32), 1) pred_mask = np.zeros((input['height'], input['width']), dtype=np.int8) for i in pred: cv2.fillPoly(pred_mask, np.array([i]).astype(np.int32), 1) gt_mask = gt_mask.astype(np.uint8) gt_mask = (gt_mask * 255).astype(np.uint8) pred_mask = pred_mask.astype(np.uint8) pred_mask = (pred_mask * 255).astype(np.uint8) def compute_iou_and_accuracy(arrs, edge_mask1): intersection = cv2.bitwise_and(arrs, edge_mask1) union = cv2.bitwise_or(arrs, edge_mask1) intersection_sum = np.sum(intersection) union_sum = np.sum(union) iou = (intersection_sum) / (union_sum) total = np.sum(arrs) correct_predictions = intersection_sum accuracy = correct_predictions / total # print(iou, accuracy) return iou, accuracy res_iou, res_accuracy = compute_iou_and_accuracy( pred_mask, gt_mask) res_ahd, res_hd, res_hd95 = assd(pred_mask, gt_mask), hd( pred_mask, gt_mask), hd95(pred_mask, gt_mask) self.ahd[categories_list[reg_type]].append(res_ahd) self.hd[categories_list[reg_type]].append(res_hd) self.hd95[categories_list[reg_type]].append(res_hd95) self.hd[categories_list[reg_type]].append( hd(pred_mask, gt_mask)) self.iou[categories_list[reg_type]].append(res_iou) self.acc[categories_list[reg_type]].append(res_accuracy) doc_ahd[categories_list[reg_type]].append(res_ahd) doc_hd[categories_list[reg_type]].append(res_hd) doc_hd95[categories_list[reg_type]].append(res_hd95) doc_hd[categories_list[reg_type]].append( hd(pred_mask, gt_mask)) doc_iou[categories_list[reg_type]].append(res_iou) doc_acc[categories_list[reg_type]].append(res_accuracy) # One has points # elif len(gt) ^ len(pred): # total_area = 0 # for each_pred in pred: # total_area += PolyArea(each_pred[:, 0], each_pred[:, 1]) # hd = total_area / 100 # self.ahd[categories_list[reg_type]].append(hd) # self.hd[categories_list[reg_type]].append(hd) # self.hd95[categories_list[reg_type]].append(hd) # self.iou[categories_list[reg_type]].append(0) # self.acc[categories_list[reg_type]].append(0) # Both Empty # elif len(gt) == 0 and len(pred) != 0: else: # self.hd[categories_list[reg_type]].append(0) pass # print("Over for doc") total_ahd = list() for l in doc_ahd.values(): total_ahd.extend(l) doc_ahd = np.mean(total_ahd) total_hd = list() for l in doc_hd.values(): total_hd.extend(l) doc_hd = np.mean(total_hd) total_hd95 = list() for l in doc_hd95.values(): total_hd95.extend(l) doc_hd95 = np.mean(total_hd95) total_iou = list() for l in doc_iou.values(): total_iou.extend(l) doc_iou = np.mean(total_iou) total_acc = list() for l in doc_acc.values(): total_acc.extend(l) doc_acc = np.mean(total_acc) self.doc_wise[input['file_name']] = { "AHD": doc_ahd, "IOU": doc_iou, "HD": doc_hd, "HD95": doc_hd95, "ACC": doc_acc }
def main(model_path, exp_config, do_plots=False): # Get Data data_loader = data_switch(exp_config.data_identifier) data = data_loader(exp_config) # Make and restore vagan model segmenter_model = segmenter(exp_config=exp_config, data=data, fixed_batch_size=1) # CRF model requires fixed batch size segmenter_model.load_weights(model_path, type='best_dice') # Run predictions in an endless loop dice_list = [] assd_list = [] hd_list = [] for ii, batch in enumerate(data.test.iterate_batches(1)): if ii % 100 == 0: logging.info("Progress: %d" % ii) x, y = batch y_ = segmenter_model.predict(x)[0] per_lbl_dice = [] per_lbl_assd = [] per_lbl_hd = [] per_pixel_preds = [] per_pixel_gts = [] if do_plots and not sys_config.running_on_gpu_host: fig = plt.figure() fig.add_subplot(131) plt.imshow(np.squeeze(x), cmap='gray') fig.add_subplot(132) plt.imshow(np.squeeze(y_)) fig.add_subplot(133) plt.imshow(np.squeeze(y)) plt.show() for lbl in range(exp_config.nlabels): binary_pred = (y_ == lbl) * 1 binary_gt = (y == lbl) * 1 if np.sum(binary_gt) == 0 and np.sum(binary_pred) == 0: per_lbl_dice.append(1) per_lbl_assd.append(0) per_lbl_hd.append(0) elif np.sum(binary_pred) > 0 and np.sum(binary_gt) == 0 or np.sum(binary_pred) == 0 and np.sum(binary_gt) > 0: logging.warning('Structure missing in either GT (x)or prediction. ASSD and HD will not be accurate.') per_lbl_dice.append(0) per_lbl_assd.append(1) per_lbl_hd.append(1) else: per_lbl_dice.append(dc(binary_pred, binary_gt)) per_lbl_assd.append(assd(binary_pred, binary_gt)) per_lbl_hd.append(hd(binary_pred, binary_gt)) dice_list.append(per_lbl_dice) assd_list.append(per_lbl_assd) hd_list.append(per_lbl_hd) per_pixel_preds.append(y_.flatten()) per_pixel_gts.append(y.flatten()) dice_arr = np.asarray(dice_list) assd_arr = np.asarray(assd_list) hd_arr = np.asarray(hd_list) mean_per_lbl_dice = dice_arr.mean(axis=0) mean_per_lbl_assd = assd_arr.mean(axis=0) mean_per_lbl_hd = hd_arr.mean(axis=0) logging.info('Dice') logging.info(structures_dict) logging.info(mean_per_lbl_dice) logging.info(np.mean(mean_per_lbl_dice)) logging.info('foreground mean: %f' % (np.mean(mean_per_lbl_dice[1:]))) logging.info('ASSD') logging.info(structures_dict) logging.info(mean_per_lbl_assd) logging.info(np.mean(mean_per_lbl_assd)) logging.info('HD') logging.info(structures_dict) logging.info(mean_per_lbl_hd) logging.info(np.mean(mean_per_lbl_hd))
def main(FLAGS): # set GPU device to use os.environ['CUDA_VISIBLE_DEVICES'] = '3' # define the experiments encoders = FLAGS.encoders.split(',') losses = FLAGS.losses.split(',') alpha = FLAGS.loss_param1.split(',') experiments = [encoders, losses, alpha] print(experiments) # get data files and sort them image_files = os.listdir(FLAGS.test_X_dir) anno_files = os.listdir(FLAGS.test_y_dir) image_files.sort() anno_files.sort() for experiment in itertools.product(*experiments): # switch to activate training session do_eval = True # load the base configurations if experiment[0] == 'UNet': configs = load_config( os.path.join(FLAGS.base_configs_dir, 'unet2d.json')) elif experiment[0] == 'UNet3D': configs = load_config( os.path.join(FLAGS.base_configs_dir, 'unet3d.json')) elif experiment[0] == 'VGG16' or experiment[0] == 'VGG19': configs = load_config( os.path.join(FLAGS.base_configs_dir, 'vgg16_unet.json')) else: configs = load_config( os.path.join(FLAGS.base_configs_dir, 'xception_unet.json')) # set the path to the model (checkpoints are fine for this) ckpt_name = 'encoder_{}_loss_{}_alpha_{}_beta_{}_ckpt.h5'.format( experiment[0], experiment[1], experiment[2], 1. - literal_eval(experiment[2])) configs['paths']['load_model'] = os.path.join(FLAGS.ckpt_dir, ckpt_name) # switch to inference configs['config_file']['type_signal'] = 'Inference' # perform some preprocessing configs['preprocessing']['categorical_switch'] = 'True' configs['preprocessing']['minimum_image_intensity'] = '0.0' configs['preprocessing']['maximum_image_intensity'] = '2048.0' configs['preprocessing']['normalization_type'] = '{}'.format( FLAGS.normalization) # set some other configurations configs['training_configurations']['batch_size'] = '{}'.format( FLAGS.batch_size) configs['config_file']['input_shape'] = '({}, {}, {})'.format( FLAGS.height, FLAGS.width, FLAGS.channels) # ensure xentropy/jaccard/focal only used once per encoder if experiment[1] == 'sparse_categorical_crossentropy'\ or experiment[1] == 'categorical_crossentropy'\ or experiment[1] == 'jaccard'\ or experiment[1] == 'focal': if experiment[1] == 'focal': configs['loss_function']['parameter1'] = '0.75' configs['loss_function']['parameter2'] = '2.0' if experiment[1] == 'jaccard': configs['loss_function']['parameter1'] = '100.0' if experiment[2] == '0.3': configs['loss_function']['loss'] = experiment[1] else: do_eval = False elif experiment[1] == 'tversky': configs['loss_function']['loss'] = experiment[1] configs['loss_function']['parameter1'] = experiment[2] configs['loss_function']['parameter2'] = str( 1. - literal_eval(experiment[2])) else: do_eval = False # create a location to store evaluation metrics metrics = np.zeros((len(image_files), FLAGS.classes, 8)) overall_accuracy = np.zeros((len(image_files), )) # create a file writer to store the metrics excel_name = '{}_{}_{}_{}_metrics.xlsx'.format( experiment[0], experiment[1], experiment[2], 1. - literal_eval(experiment[2])) writer = pd.ExcelWriter(excel_name) for i in range(len(image_files)): K.clear_session() # define path to the test data configs['paths']['test_X'] = os.path.join(FLAGS.test_X_dir, image_files[i]) if do_eval is True: configs_lvl1, errors_lvl1, warnings_lvl1 = level_one_error_checking( configs) if any(warnings_lvl1): with open('errors.txt', 'a') as f: for warning in warnings_lvl1: f.write("%s\n" % warning) f.close() print('Level 1 warnings encountered.') print( "The following level 1 warnings were identified and corrected based on engine defaults:" ) for warning in warnings_lvl1: print(warning) if any(errors_lvl1): print('Level 1 errors encountered.') print( "Please fix the level 1 errors below before continuing:" ) for error in errors_lvl1: print(error) else: configs_lvl2, errors_lvl2, warnings_lvl2 = level_two_error_checking( configs_lvl1) if any(warnings_lvl2): print('Level 2 warnings encountered.') print( "The following level 2 warnings were identified and corrected based on engine defaults:" ) for warning in warnings_lvl2: print(warning) if any(errors_lvl2): print('Level 2 errors encountered.') print( "Please fix the level 2 errors below before continuing:" ) for error in errors_lvl2: print(error) else: engine = Dlae(configs) engine.run() if any(engine.errors): print('Level 3 errors encountered.') print( "Please fix the level 3 errors below before continuing:" ) for error in engine.errors: print(error) # check if the images and annotations are the correct files print(image_files[i], anno_files[i]) pred_file = glob( os.path.join(FLAGS.predictions_temp_dir, '*.h5'))[0] pt_name = image_files[i].split('.')[0] new_name_raw = pt_name + '_{}_{}_{}_{}_raw.h5'.format( experiment[0], experiment[1], experiment[2], 1. - literal_eval(experiment[2])) new_file_raw = os.path.join(FLAGS.predictions_final_dir, new_name_raw) os.rename(pred_file, new_file_raw) ref = read_hdf5_multientry( os.path.join(FLAGS.test_y_dir, anno_files[i])) ref = np.squeeze(np.asarray(ref)) preds = read_hdf5(new_file_raw) if experiment[0] == 'UNet3D': ref = np.transpose(ref, (1, 2, 0)) # stich the image back together first sw = SlidingWindow(ref, [96, 96, 40], [128, 128, 48]) preds = sw.stitch_patches(preds, sw.window_corner_coords, [128, 128, 48], sw.img_shape, FLAGS.classes) preds = np.argmax(preds, axis=-1) else: preds = np.argmax(preds, axis=-1) overall_accuracy[i] = skm.accuracy_score( ref.flatten(), preds.flatten()) for j in range(FLAGS.classes): organ_pred = (preds == j).astype(np.int64) organ_ref = (ref == j).astype(np.int64) if np.sum(organ_pred) == 0 or np.sum(organ_ref) == 0: metrics[i, j, 0] = 0. metrics[i, j, 1] = 0. metrics[i, j, 2] = 1. metrics[i, j, 3] = 0. metrics[i, j, 4] = 0. metrics[i, j, 5] = 0. metrics[i, j, 6] = np.inf metrics[i, j, 7] = np.inf else: metrics[i, j, 0] = jaccard_index(organ_ref, organ_pred) metrics[i, j, 1] = dice_similarity_coefficient( organ_ref, organ_pred) metrics[i, j, 2] = relative_volume_difference( organ_ref, organ_pred) metrics[i, j, 3] = precision(organ_ref, organ_pred) metrics[i, j, 4] = recall(organ_ref, organ_pred) metrics[i, j, 5] = matthews_correlation_coefficient( organ_ref, organ_pred) metrics[i, j, 6] = mpm.hd95(organ_pred, organ_ref) metrics[i, j, 7] = mpm.assd(organ_pred, organ_ref) print(overall_accuracy[i]) print(metrics[i]) else: pass if do_eval is True: for k in range(metrics.shape[-1]): data = pd.DataFrame( metrics[:, :, k], columns=['bg', 'pros', 'eus', 'sv', 'rect', 'blad']) data.to_excel(writer, sheet_name=str(k)) acc = pd.DataFrame(overall_accuracy, columns=['acc']) acc.to_excel(writer, sheet_name='acc') writer.save()
# ============================================================================= # Distancia media entre superficies # ============================================================================= distanceASD = mdm.asd(cHull_binary,cHull_Mbinary,connectivity=1) print('Distancia Media',distanceASD) # ============================================================================= # Distancia media simétrica entre superficies # ============================================================================= distanceASSD = mdm.assd(cHull_binary,cHull_Mbinary,connectivity=1) print('Distancia Media Simétrica',distanceASSD) # ============================================================================= # Distancia media entre superficie de objetos # ============================================================================= distanceObjASD = mdm.obj_asd(cHull_binary,cHull_Mbinary,connectivity=1) print('Distancia Media entre superficie de objetos: ',distanceObjASD) # =============================================================================
for j_c in range(n_class): dice_c[i_c, j_c] = dk_seg.dice(batch_y[0, :, :, :, j_c] == 1, y_tr_pr_c == j_c) y_t_c = np.argmax( batch_y[0, :, :, :, :], axis=-1) #dk_aux.save_pred_thumbs(batch_x[0,:,:,:,0], y_t_c, y_tr_pr_c, False, i_c, i_eval, images_dir ) '''if i_eval==0: save_pred_mhds(batch_x[0,:,:,:,0], y_t_c, y_tr_pr_c, False, i_c, i_eval) else: save_pred_mhds(None, None, y_tr_pr_c, False, i_c, i_eval)''' dice_c[i_c, n_class] = hd95(y_t_c, y_tr_pr_c) dice_c[i_c, n_class+1] = asd(y_t_c, y_tr_pr_c) dice_c[i_c, n_class+2] = assd(y_t_c, y_tr_pr_c) y_tr_pr_soft= y_tr_pr_sum[:,:,:,1]/(y_tr_pr_cnt+1e-10) #dk_aux.save_pred_soft_thumbs(batch_x[0,:,:,:,0], y_t_c, y_tr_pr_c, y_tr_pr_soft, False, i_c, i_eval, images_dir) error_mask= dk_aux.seg_2_anulus(y_t_c, radius= 2.0) plot_save_path= None ECE, MCE, ECE_curve= dk_aux.estimate_ECE_and_MCE(y_t_c, y_tr_pr_soft, plot_save_path=plot_save_path) dice_c[i_c, n_class+3]= ECE dice_c[i_c, n_class+4]= MCE plot_save_path= None ECE, MCE, ECE_curve= dk_aux.estimate_ECE_and_MCE_masked(y_t_c, y_tr_pr_soft, error_mask, plot_save_path=plot_save_path) dice_c[i_c, n_class+5]= ECE dice_c[i_c, n_class+6]= MCE
def main(input_folder, output_folder, model_path, exp_config, do_postprocessing=False, gt_exists=True): # Get Data data_loader = data_switch(exp_config.data_identifier) data = data_loader(exp_config) # Make and restore vagan model segmenter_model = segmenter( exp_config=exp_config, data=data, fixed_batch_size=1) # CRF model requires fixed batch size segmenter_model.load_weights(model_path, type='best_dice') total_time = 0 total_volumes = 0 dice_list = [] assd_list = [] hd_list = [] for folder in os.listdir(input_folder): folder_path = os.path.join(input_folder, folder) if os.path.isdir(folder_path): infos = {} for line in open(os.path.join(folder_path, 'Info.cfg')): label, value = line.split(':') infos[label] = value.rstrip('\n').lstrip(' ') patient_id = folder.lstrip('patient') if not int(patient_id) % 5 == 0: continue ED_frame = int(infos['ED']) ES_frame = int(infos['ES']) for file in glob.glob( os.path.join(folder_path, 'patient???_frame??.nii.gz')): logging.info(' ----- Doing image: -------------------------') logging.info('Doing: %s' % file) logging.info(' --------------------------------------------') file_base = file.split('.nii.gz')[0] frame = int(file_base.split('frame')[-1]) img, img_affine, img_header = utils.load_nii(file) img = utils.normalise_image(img) zooms = img_header.get_zooms() if gt_exists: file_mask = file_base + '_gt.nii.gz' mask, mask_affine, mask_header = utils.load_nii(file_mask) start_time = time.time() if exp_config.dimensionality_mode == '2D': pixel_size = (img_header.structarr['pixdim'][1], img_header.structarr['pixdim'][2]) scale_vector = (pixel_size[0] / exp_config.target_resolution[0], pixel_size[1] / exp_config.target_resolution[1]) predictions = [] nx, ny = exp_config.image_size for zz in range(img.shape[2]): slice_img = np.squeeze(img[:, :, zz]) slice_rescaled = transform.rescale(slice_img, scale_vector, order=1, preserve_range=True, multichannel=False, mode='constant') x, y = slice_rescaled.shape x_s = (x - nx) // 2 y_s = (y - ny) // 2 x_c = (nx - x) // 2 y_c = (ny - y) // 2 # Crop section of image for prediction if x > nx and y > ny: slice_cropped = slice_rescaled[x_s:x_s + nx, y_s:y_s + ny] else: slice_cropped = np.zeros((nx, ny)) if x <= nx and y > ny: slice_cropped[x_c:x_c + x, :] = slice_rescaled[:, y_s:y_s + ny] elif x > nx and y <= ny: slice_cropped[:, y_c:y_c + y] = slice_rescaled[x_s:x_s + nx, :] else: slice_cropped[x_c:x_c + x, y_c:y_c + y] = slice_rescaled[:, :] # GET PREDICTION network_input = np.float32( np.tile(np.reshape(slice_cropped, (nx, ny, 1)), (1, 1, 1, 1))) mask_out, softmax = segmenter_model.predict( network_input) prediction_cropped = np.squeeze(softmax[0, ...]) # ASSEMBLE BACK THE SLICES slice_predictions = np.zeros( (x, y, exp_config.nlabels)) # insert cropped region into original image again if x > nx and y > ny: slice_predictions[x_s:x_s + nx, y_s:y_s + ny, :] = prediction_cropped else: if x <= nx and y > ny: slice_predictions[:, y_s:y_s + ny, :] = prediction_cropped[ x_c:x_c + x, :, :] elif x > nx and y <= ny: slice_predictions[ x_s:x_s + nx, :, :] = prediction_cropped[:, y_c:y_c + y, :] else: slice_predictions[:, :, :] = prediction_cropped[ x_c:x_c + x, y_c:y_c + y, :] # RESCALING ON THE LOGITS if gt_exists: prediction = transform.resize( slice_predictions, (mask.shape[0], mask.shape[1], exp_config.nlabels), order=1, preserve_range=True, mode='constant') else: # This can occasionally lead to wrong volume size, therefore if gt_exists # we use the gt mask size for resizing. prediction = transform.rescale( slice_predictions, (1.0 / scale_vector[0], 1.0 / scale_vector[1], 1), order=1, preserve_range=True, multichannel=False, mode='constant') prediction = np.uint8(np.argmax(prediction, axis=-1)) # import matplotlib.pyplot as plt # fig = plt.Figure() # for ii in range(3): # plt.subplot(1, 3, ii + 1) # plt.imshow(np.squeeze(prediction)) # plt.show() predictions.append(prediction) prediction_arr = np.transpose( np.asarray(predictions, dtype=np.uint8), (1, 2, 0)) elif exp_config.dimensionality_mode == '3D': nx, ny, nz = exp_config.image_size pixel_size = (img_header.structarr['pixdim'][1], img_header.structarr['pixdim'][2], img_header.structarr['pixdim'][3]) scale_vector = (pixel_size[0] / exp_config.target_resolution[0], pixel_size[1] / exp_config.target_resolution[1], pixel_size[2] / exp_config.target_resolution[2]) vol_scaled = transform.rescale(img, scale_vector, order=1, preserve_range=True, multichannel=False, mode='constant') nz_max = exp_config.image_size[2] slice_vol = np.zeros((nx, ny, nz_max), dtype=np.float32) nz_curr = vol_scaled.shape[2] stack_from = (nz_max - nz_curr) // 2 stack_counter = stack_from x, y, z = vol_scaled.shape x_s = (x - nx) // 2 y_s = (y - ny) // 2 x_c = (nx - x) // 2 y_c = (ny - y) // 2 for zz in range(nz_curr): slice_rescaled = vol_scaled[:, :, zz] if x > nx and y > ny: slice_cropped = slice_rescaled[x_s:x_s + nx, y_s:y_s + ny] else: slice_cropped = np.zeros((nx, ny)) if x <= nx and y > ny: slice_cropped[x_c:x_c + x, :] = slice_rescaled[:, y_s:y_s + ny] elif x > nx and y <= ny: slice_cropped[:, y_c:y_c + y] = slice_rescaled[x_s:x_s + nx, :] else: slice_cropped[x_c:x_c + x, y_c:y_c + y] = slice_rescaled[:, :] slice_vol[:, :, stack_counter] = slice_cropped stack_counter += 1 stack_to = stack_counter network_input = np.float32( np.reshape(slice_vol, (1, nx, ny, nz_max, 1))) start_time = time.time() mask_out, softmax = segmenter_model.predict(network_input) logging.info('Classified 3D: %f secs' % (time.time() - start_time)) prediction_nzs = mask_out[0, :, :, stack_from: stack_to] # non-zero-slices if not prediction_nzs.shape[2] == nz_curr: raise ValueError('sizes mismatch') # ASSEMBLE BACK THE SLICES prediction_scaled = np.zeros( vol_scaled.shape) # last dim is for logits classes # insert cropped region into original image again if x > nx and y > ny: prediction_scaled[x_s:x_s + nx, y_s:y_s + ny, :] = prediction_nzs else: if x <= nx and y > ny: prediction_scaled[:, y_s:y_s + ny, :] = prediction_nzs[x_c:x_c + x, :, :] elif x > nx and y <= ny: prediction_scaled[ x_s:x_s + nx, :, :] = prediction_nzs[:, y_c:y_c + y, :] else: prediction_scaled[:, :, :] = prediction_nzs[ x_c:x_c + x, y_c:y_c + y, :] logging.info('Prediction_scaled mean %f' % (np.mean(prediction_scaled))) prediction = transform.resize( prediction_scaled, (mask.shape[0], mask.shape[1], mask.shape[2], 1), order=1, preserve_range=True, mode='constant') prediction = np.argmax(prediction, axis=-1) prediction_arr = np.asarray(prediction, dtype=np.uint8) # This is the same for 2D and 3D again if do_postprocessing: prediction_arr = utils.keep_largest_connected_components( prediction_arr) elapsed_time = time.time() - start_time total_time += elapsed_time total_volumes += 1 logging.info('Evaluation of volume took %f secs.' % elapsed_time) if frame == ED_frame: frame_suffix = '_ED' elif frame == ES_frame: frame_suffix = '_ES' else: raise ValueError( 'Frame doesnt correspond to ED or ES. frame = %d, ED = %d, ES = %d' % (frame, ED_frame, ES_frame)) # Save prediced mask out_file_name = os.path.join( output_folder, 'prediction', 'patient' + patient_id + frame_suffix + '.nii.gz') if gt_exists: out_affine = mask_affine out_header = mask_header else: out_affine = img_affine out_header = img_header logging.info('saving to: %s' % out_file_name) utils.save_nii(out_file_name, prediction_arr, out_affine, out_header) # Save image data to the same folder for convenience image_file_name = os.path.join( output_folder, 'image', 'patient' + patient_id + frame_suffix + '.nii.gz') logging.info('saving to: %s' % image_file_name) utils.save_nii(image_file_name, img, out_affine, out_header) if gt_exists: # Save GT image gt_file_name = os.path.join( output_folder, 'ground_truth', 'patient' + patient_id + frame_suffix + '.nii.gz') logging.info('saving to: %s' % gt_file_name) utils.save_nii(gt_file_name, mask, out_affine, out_header) # Save difference mask between predictions and ground truth difference_mask = np.where( np.abs(prediction_arr - mask) > 0, [1], [0]) difference_mask = np.asarray(difference_mask, dtype=np.uint8) diff_file_name = os.path.join( output_folder, 'difference', 'patient' + patient_id + frame_suffix + '.nii.gz') logging.info('saving to: %s' % diff_file_name) utils.save_nii(diff_file_name, difference_mask, out_affine, out_header) # calculate metrics y_ = prediction_arr y = mask per_lbl_dice = [] per_lbl_assd = [] per_lbl_hd = [] for lbl in [3, 1, 2]: #range(exp_config.nlabels): binary_pred = (y_ == lbl) * 1 binary_gt = (y == lbl) * 1 if np.sum(binary_gt) == 0 and np.sum(binary_pred) == 0: per_lbl_dice.append(1) per_lbl_assd.append(0) per_lbl_hd.append(0) elif np.sum(binary_pred) > 0 and np.sum( binary_gt) == 0 or np.sum( binary_pred) == 0 and np.sum(binary_gt) > 0: logging.warning( 'Structure missing in either GT (x)or prediction. ASSD and HD will not be accurate.' ) per_lbl_dice.append(0) per_lbl_assd.append(1) per_lbl_hd.append(1) else: per_lbl_dice.append(dc(binary_pred, binary_gt)) per_lbl_assd.append( assd(binary_pred, binary_gt, voxelspacing=zooms)) per_lbl_hd.append( hd(binary_pred, binary_gt, voxelspacing=zooms)) dice_list.append(per_lbl_dice) assd_list.append(per_lbl_assd) hd_list.append(per_lbl_hd) logging.info('Average time per volume: %f' % (total_time / total_volumes)) dice_arr = np.asarray(dice_list) assd_arr = np.asarray(assd_list) hd_arr = np.asarray(hd_list) mean_per_lbl_dice = dice_arr.mean(axis=0) mean_per_lbl_assd = assd_arr.mean(axis=0) mean_per_lbl_hd = hd_arr.mean(axis=0) logging.info('Dice') logging.info(mean_per_lbl_dice) logging.info(np.mean(mean_per_lbl_dice)) logging.info('ASSD') logging.info(mean_per_lbl_assd) logging.info(np.mean(mean_per_lbl_assd)) logging.info('HD') logging.info(mean_per_lbl_hd) logging.info(np.mean(mean_per_lbl_hd))