def update_eval_metrics(preds, labels, eval_metrics): if len(labels.shape) == 2: preds = np.expand_dims(preds, axis=0) labels = np.expand_dims(labels, axis=0) N = labels.shape[0] for i in range(N): pred = preds[i, :, :] label = labels[i, :, :] eval_metrics['dice score'].append(dc(pred, label)) eval_metrics['precision'].append(precision(pred, label)) eval_metrics['recall'].append(recall(pred, label)) eval_metrics['sensitivity'].append(sensitivity(pred, label)) eval_metrics['specificity'].append(specificity(pred, label)) if np.sum(pred) > 0 and np.sum(label) > 0: eval_metrics['hausdorff'].append(hd(pred, label)) eval_metrics['hausdorff 95%'].append(hd95(pred, label)) eval_metrics['asd'].append(asd(pred, label)) eval_metrics['assd'].append(assd(pred, label)) eval_metrics['jaccard'].append(jc(pred, label)) else: eval_metrics['hausdorff'].append('nan') eval_metrics['hausdorff 95%'].append('nan') eval_metrics['asd'].append('nan') eval_metrics['assd'].append('nan') eval_metrics['jaccard'].append('nan') return eval_metrics
def __compute_subcortical_structures_location(self, volume, spacing, category=None, reference='BCB'): distances = {} overlaps = {} distances_columns = [] overlaps_columns = [] tract_cutoff = 0.5 if reference == 'BrainLab': tract_cutoff = 0.25 tracts_dict = ResourcesConfiguration.getInstance().subcortical_structures['MNI'][reference] for i, tfn in enumerate(tracts_dict.keys()): reg_tract_ni = nib.load(tracts_dict[tfn]) reg_tract = reg_tract_ni.get_data()[:] reg_tract[reg_tract < tract_cutoff] = 0 reg_tract[reg_tract >= tract_cutoff] = 1 overlap_volume = np.logical_and(reg_tract, volume).astype('uint8') distances_columns.append('distance_' + tfn.split('.')[0][:-4] + '_' + category) overlaps_columns.append('overlap_' + tfn.split('.')[0][:-4] + '_' + category) if np.count_nonzero(overlap_volume) != 0: distances[tfn] = -1. overlaps[tfn] = (np.count_nonzero(overlap_volume) / np.count_nonzero(volume)) * 100. else: dist = -1. if np.count_nonzero(reg_tract) > 0: dist = hd95(volume, reg_tract, voxelspacing=reg_tract_ni.header.get_zooms(), connectivity=1) distances[tfn] = dist overlaps[tfn] = 0. self.diagnosis_parameters.statistics[category]['Overall'].mni_space_subcortical_structures_overlap[reference] = overlaps self.diagnosis_parameters.statistics[category]['Overall'].mni_space_subcortical_structures_distance[reference] = distances
def hd95_multiply(logits, targets, cls_id=None): if isinstance(logits, torch.Tensor): logits = logits.cpu().numpy() targets = targets.cpu().numpy() batch_size, class_cnt = logits.shape[0], logits.shape[1] #print('batch_size, class_cnt ', batch_size, class_cnt ) hd95_score = [] for class_index in range(class_cnt): #print(class_index, np.unique(logits.argmax(axis=1))) predict = logits.argmax(axis=-3) == class_index target = (targets == class_index) #print('predict, target', class_index, predict.shape, target.shape, #predict.sum(), target.sum(), np.unique(targets)) score = [ hd95(res, ref) for res, ref in zip(predict, target) if np.count_nonzero(res) > 0 and np.count_nonzero(ref) > 0 ] #print('class_index=', class_index, score) hd95_score.append(np.mean(score)) #print('hd95_score', hd95_score) if cls_id is None: return torch.Tensor(hd95_score) else: return torch.Tensor(hd95_score)[cls_id]
def Hausdorff(pred, gt): batch_s = pred.size(0) HDC = np.zeros(batch_s) pred[pred > 0] = 1 pred = pred.detach().cpu().numpy() gt[gt > 0] = 1 gt = gt.detach().cpu().numpy() for jj in range(batch_s): if pred[jj, :, :].sum() > 0 and gt[jj, :, :].sum() > 0: HDC[jj] = binary.hd95(pred[jj, :, :], gt[jj, :, :]) else: HDC[jj] = 0 return HDC
def __compute_multifocality(self, volume, spacing): multifocality = None multifocal_elements = None multifocal_largest_minimum_distance = None tumor_clusters = measurements.label(volume)[0] tumor_clusters_labels = regionprops(tumor_clusters) if len(tumor_clusters_labels) > 1: multifocality = False multifocal_elements = 0 # Computing the radius of the largest component. radiuses = [] parts_labels = [] for l in range(len(tumor_clusters_labels)): volume_ml = np.count_nonzero(tumor_clusters == (l + 1)) * np.prod(spacing[0:3]) * 1e-3 if volume_ml >= 0.1: # Discarding tumor parts smaller than 0.1 ml multifocal_elements = multifocal_elements + 1 radiuses.append(tumor_clusters_labels[l].equivalent_diameter / 2.) parts_labels.append(l) max_radius = np.max(radiuses) max_radius_index = parts_labels[radiuses.index(max_radius)] # Computing the minimum distances between each foci main_tumor_label = np.zeros(volume.shape) main_tumor_label[tumor_clusters == (max_radius_index + 1)] = 1 for l, lab in enumerate(parts_labels): if lab != max_radius_index: satellite_label = np.zeros(volume.shape) satellite_label[tumor_clusters == (lab + 1)] = 1 dist = hd95(satellite_label, main_tumor_label, voxelspacing=spacing, connectivity=1) if multifocal_largest_minimum_distance is None: multifocal_largest_minimum_distance = dist elif dist > multifocal_largest_minimum_distance: multifocal_largest_minimum_distance = dist if multifocal_largest_minimum_distance >= 5.0: multifocality = True else: multifocality = False multifocal_elements = 1 multifocal_largest_minimum_distance = -1. self.diagnosis_parameters.tumor_multifocal = multifocality self.diagnosis_parameters.tumor_parts = multifocal_elements self.diagnosis_parameters.tumor_multifocal_distance = multifocal_largest_minimum_distance
def run_hd(image1, image2, mode='max'): image1_data, hd1 = nrrd.read(image1) space_x = np.abs(hd1['space directions'][0, 0]) space_y = np.abs(hd1['space directions'][1, 1]) space_z = np.abs(hd1['space directions'][2, 2]) image2_data, hd2 = nrrd.read(image2) if (hd1['space origin'] == hd2['space origin']).all(): if mode == 'max': hd_val = hd(image1_data, image2_data, (space_x, space_y, space_z)) elif mode == '95': hd_val = hd95(image1_data, image2_data, (space_x, space_y, space_z)) else: raise Exception( 'Unknown mode "{}". Possible values are "max" or "95".'.format( mode)) else: hd_val = None return hd_val
def channelwise_hd95(pred, target): """ calculate channel-wise 95 percentile symmetric Hausdorff distance, especially for pred with multiple channels. :param pred: ndarray with size [C, H, W], predicted boundary :param target: ndarray with size [C, H, W], GT boundary :return mean_hd95: float, average hdf """ max_hd95 = 2 * target.shape[1] channel_res = [] for c_inx in range(0, len(pred)): if np.sum(target[c_inx]) != 0: if np.sum(pred[c_inx]) == 0: res = max_hd95 else: res = hd95(pred[c_inx], target[c_inx]) channel_res.append(res) mean_hd95 = sum(channel_res) / len(channel_res) return mean_hd95
def hausdorff95(prediction: torch.Tensor, target: torch.Tensor, merge_operation: Callable[ [torch.Tensor], float] = torch.max) -> torch.Tensor: """ 95th percentile of the Hausdorff Distance. Computes the 95th percentile of the (symmetric) Hausdorff Distance (HD) between the binary objects in two images. Heavily depends on medpy.metric.binary.hd95, so check it out for more details. Args: prediction: Network output. Dimensions - (Batch, Class, Depth, Height, Width) target: Target values. Dimensions - (Batch, Class, Depth, Height, Width) merge_operation: Operation to merge results from all the slices in the batch into a single value. Should be a function that accepts two dimensional tensors (Batch, Depth). Good examples are: torch.mean, torch.max, torch.min. Returns: torch.Tensor: Hausdorff distance for the provided volume """ assert utils.is_binary(prediction), "Predictions must be binary" assert utils.is_binary(target), "Target must be binary" prediction = prediction.cpu().detach().numpy() target = target.cpu().detach().numpy() volumes_count, _, slices_count, _, _ = prediction.shape results = np.zeros((volumes_count, slices_count)) for volume_idx in range(volumes_count): for slice_idx in range(slices_count): prediction_slice = prediction[ volume_idx, FIRST_CLASS, slice_idx, ...] target_slice = target[volume_idx, FIRST_CLASS, slice_idx, ...] if utils.has_only_zeros(prediction_slice) or \ utils.has_only_zeros(target_slice): results[volume_idx, slice_idx] = 0 else: results[volume_idx, slice_idx] = mp.hd95(prediction_slice, target_slice) return merge_operation(torch.from_numpy(results))
def slicewise_hd95(pred, target, n_classes=3): """ calculate Average Hausdorff distance between pred and target of single image :param pred: ndarray with size [H, W], predicted bound :param target: ndarray with size [H, W], ground truth :param n_classes: int, # of classes """ max_hd95 = 2 * target.shape[0] slice_res = [] for c_inx in range(1, n_classes): pred_cinx = (pred == c_inx) target_cinx = (target == c_inx) if np.sum(target_cinx) != 0: if np.sum(pred_cinx) == 0: res = max_hd95 else: res = hd95(pred_cinx, target_cinx) slice_res.append(res) mean_res = sum(slice_res) / len(slice_res) return mean_res
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model-file', type=str, default='./logs/test1/20190506_221021.177567/checkpoint_200.pth.tar', help='Model path') parser.add_argument('--datasetTest', type=list, default=[1], help='test folder id contain images ROIs to test') parser.add_argument('--dataset', type=str, default='test', help='test folder id contain images ROIs to test') parser.add_argument('-g', '--gpu', type=int, default=0) parser.add_argument('--data-dir', default='../../../../Dataset/Fundus/', help='data root path') parser.add_argument('--out-stride', type=int, default=16, help='out-stride of deeplabv3+',) parser.add_argument('--sync-bn', type=bool, default=False, help='sync-bn in deeplabv3+') parser.add_argument('--freeze-bn', type=bool, default=False, help='freeze batch normalization of deeplabv3+') parser.add_argument('--movingbn', type=bool, default=False, help='moving batch normalization of deeplabv3+ in the test phase',) parser.add_argument('--test-prediction-save-path', type=str, default='./results/rebuttle-0401/', help='Path root for test image and mask') args = parser.parse_args() os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu) model_file = args.model_file output_path = os.path.join(args.test_prediction_save_path, 'test' + str(args.datasetTest[0]), args.model_file.split('/')[-2]) # 1. dataset composed_transforms_test = transforms.Compose([ tr.Normalize_tf(), tr.ToTensor() ]) db_test = DL.FundusSegmentation(base_dir=args.data_dir, phase='test', splitid=args.datasetTest, transform=composed_transforms_test, state='prediction') batch_size = 12 test_loader = DataLoader(db_test, batch_size=batch_size, shuffle=False, num_workers=1) # 2. model model = DeepLab(num_classes=2, backbone='mobilenet', output_stride=args.out_stride, sync_bn=args.sync_bn, freeze_bn=args.freeze_bn).cuda() if torch.cuda.is_available(): model = model.cuda() print('==> Loading %s model file: %s' % (model.__class__.__name__, model_file)) # model_data = torch.load(model_file) checkpoint = torch.load(model_file) pretrained_dict = checkpoint['model_state_dict'] model_dict = model.state_dict() # 1. filter out unnecessary keys pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict} # 2. overwrite entries in the existing state dict model_dict.update(pretrained_dict) # 3. load the new state dict model.load_state_dict(model_dict) if args.movingbn: model.train() else: model.eval() val_cup_dice = 0.0 val_disc_dice = 0.0 total_hd_OC = 0.0 total_hd_OD = 0.0 total_asd_OC = 0.0 total_asd_OD = 0.0 timestamp_start = datetime.now(pytz.timezone('Asia/Hong_Kong')) total_num = 0 OC = [] OD = [] for batch_idx, (sample) in tqdm.tqdm(enumerate(test_loader),total=len(test_loader),ncols=80, leave=False): data = sample['image'] target = sample['label'] img_name = sample['img_name'] if torch.cuda.is_available(): data, target = data.cuda(), target.cuda() data, target = Variable(data), Variable(target) prediction, dc, sel, _ = model(data) prediction = torch.nn.functional.interpolate(prediction, size=(target.size()[2], target.size()[3]), mode="bilinear") data = torch.nn.functional.interpolate(data, size=(target.size()[2], target.size()[3]), mode="bilinear") target_numpy = target.data.cpu() imgs = data.data.cpu() hd_OC = 100 asd_OC = 100 hd_OD = 100 asd_OD = 100 for i in range(prediction.shape[0]): prediction_post = postprocessing(prediction[i], dataset=args.dataset) cup_dice, disc_dice = dice_coeff_2label(prediction_post, target[i]) OC.append(cup_dice) OD.append(disc_dice) if np.sum(prediction_post[0, ...]) < 1e-4: hd_OC = 100 asd_OC = 100 else: hd_OC = binary.hd95(np.asarray(prediction_post[0, ...], dtype=np.bool), np.asarray(target_numpy[i, 0, ...], dtype=np.bool)) asd_OC = binary.asd(np.asarray(prediction_post[0, ...], dtype=np.bool), np.asarray(target_numpy[i, 0, ...], dtype=np.bool)) if np.sum(prediction_post[0, ...]) < 1e-4: hd_OD = 100 asd_OD = 100 else: hd_OD = binary.hd95(np.asarray(prediction_post[1, ...], dtype=np.bool), np.asarray(target_numpy[i, 1, ...], dtype=np.bool)) asd_OD = binary.asd(np.asarray(prediction_post[1, ...], dtype=np.bool), np.asarray(target_numpy[i, 1, ...], dtype=np.bool)) val_cup_dice += cup_dice val_disc_dice += disc_dice total_hd_OC += hd_OC total_hd_OD += hd_OD total_asd_OC += asd_OC total_asd_OD += asd_OD total_num += 1 for img, lt, lp in zip([imgs[i]], [target_numpy[i]], [prediction_post]): img, lt = utils.untransform(img, lt) save_per_img(img.numpy().transpose(1, 2, 0), output_path, img_name[i], lp, lt, mask_path=None, ext="bmp") print('OC:', OC) print('OD:', OD) import csv with open('Dice_results.csv', 'a+') as result_file: wr = csv.writer(result_file, dialect='excel') for index in range(len(OC)): wr.writerow([OC[index], OD[index]]) val_cup_dice /= total_num val_disc_dice /= total_num total_hd_OC /= total_num total_asd_OC /= total_num total_hd_OD /= total_num total_asd_OD /= total_num print('''\n==>val_cup_dice : {0}'''.format(val_cup_dice)) print('''\n==>val_disc_dice : {0}'''.format(val_disc_dice)) print('''\n==>average_hd_OC : {0}'''.format(total_hd_OC)) print('''\n==>average_hd_OD : {0}'''.format(total_hd_OD)) print('''\n==>ave_asd_OC : {0}'''.format(total_asd_OC)) print('''\n==>average_asd_OD : {0}'''.format(total_asd_OD)) with open(osp.join(output_path, '../test' + str(args.datasetTest[0]) + '_log.csv'), 'a') as f: elapsed_time = ( datetime.now(pytz.timezone('Asia/Hong_Kong')) - timestamp_start).total_seconds() log = [['batch-size: '] + [batch_size] + [args.model_file] + ['cup dice coefficence: '] + \ [val_cup_dice] + ['disc dice coefficence: '] + \ [val_disc_dice] + ['average_hd_OC: '] + \ [total_hd_OC] + ['average_hd_OD: '] + \ [total_hd_OD] + ['ave_asd_OC: '] + \ [total_asd_OC] + ['average_asd_OD: '] + \ [total_asd_OD] + [elapsed_time]] log = map(str, log) f.write(','.join(log) + '\n')
def bench_pytorch_fp32(): ### Load PyTorch model pt_model = fetch_model(modelname="resunet", num_channels=1, num_classes=2, num_filters=16) checkpoint = torch.load(pytorch_model_path, map_location=torch.device('cpu')) pt_model.load_state_dict(checkpoint["model_state_dict"]) ### Run PyTorch Inference print(f"\n Starting PyTorch inference with {pytorch_model_path} ...") _ = pt_model.eval() pt_stats = [] with torch.no_grad(): for i, row in tqdm(dataset_df.iterrows()): sub_id = row[sub_idx] input_path = row[input_path_idx] mask_path = row[mask_path_idx] try: mask_image = get_mask_image(mask_path) input_image, patient_nib = get_input_image(input_path) i_start = timer() pt_output = pt_model(input_image) i_end = timer() p_start = timer() pt_output = pt_output.cpu().numpy()[0][0] pt_to_save = postprocess_output(pt_output, patient_nib.shape) p_end = timer() pt_dice_score = dice(pt_to_save, mask_image) pt_hdorff_score = hd95(pt_to_save, mask_image) pt_stat = [ i, sub_id, pt_hdorff_score, i_end - i_start, p_end - p_start ] pt_stats.append(pt_stat) except Exception as e: print(f" Inference Failed: {sub_id} ") print(e) print(f"Done PyTorch inference with {pytorch_model_path} ...") pt_stats_df = pd.DataFrame(pt_stats) date_time_str = datetime.now().strftime("%b-%d-%Y_%H-%M-%S") csv_name = f"pt_stats_{date_time_str}.csv" pt_stats_df.to_csv(csv_name, sep=',', header=False, index=False) print(f"Saved {csv_name} ...") print( f"\n PyTorch Hausdorff Distance Mean: {pt_stats_df[:][2].mean():.5f}") print( f"PyTorch Total Inf Time: {pt_stats_df[:][3].sum():.2f} sec, Mean: {pt_stats_df[:][3].mean():.2f} sec" ) return pt_stats_df
def bench_ov_int8(): # #### Load INT8 OpenVINO model ov_model_dir = brainmage_root / 'openvino/int8_openvino_model' modelname = "resunet_ma_int8" model_xml = f'{ov_model_dir}/{modelname}.xml' # Load network to the plugin ie = IECore() net = ie.read_network(model=model_xml) exec_net = ie.load_network(network=net, device_name="CPU") del net input_layer = next(iter(exec_net.input_info)) output_layer = next(iter(exec_net.outputs)) # #### Run OpenVINO Inference print(f"Starting OpenVINO inference with {ov_model_dir} ...") ov_int8_stats = [] for i, row in tqdm(dataset_df.iterrows()): sub_id = row[sub_idx] input_path = row[input_path_idx] mask_path = row[mask_path_idx] try: mask_image = get_mask_image(mask_path) input_image, patient_nib = get_input_image(input_path) i_start = timer() ov_output = exec_net.infer(inputs={input_layer: input_image}) i_end = timer() p_start = timer() ov_output = ov_output[output_layer][0][0] ov_to_save = postprocess_output(ov_output, patient_nib.shape) p_end = timer() ov_dice_score = dice(ov_to_save, mask_image) ov_hdorff_score = hd95(ov_to_save, mask_image) ov_stat = [ i, sub_id, ov_hdorff_score, i_end - i_start, p_end - p_start ] ov_int8_stats.append(ov_stat) except: print(f" Inference Failed: {sub_id} ") print(f"Done OpenVINO inference with {ov_model_dir} ...") ov_int8_stats_df = pd.DataFrame(ov_int8_stats) date_time_str = datetime.now().strftime("%b-%d-%Y_%H-%M-%S") csv_name = f"ov_int8_stats_{date_time_str}.csv" ov_int8_stats_df.to_csv(csv_name, sep=',', header=False, index=False) print(f"Saved {csv_name} ...") print( f"\n OpenVINO INT8 Hausdorff Distance Mean: {ov_int8_stats_df[:][2].mean():.5f}" ) print( f"OpenVINO INT8 Total Inf Time: {ov_int8_stats_df[:][3].sum():.2f} sec, Mean: {ov_int8_stats_df[:][3].mean():.2f} sec" ) return ov_int8_stats_df
def get_hd_assd(self, b): hd95_dist = hd95(self.pred_mid_rl[b], self.gt_mid[b]) assd_dist = assd(self.pred_mid_rl[b], self.gt_mid[b]) return hd95_dist * self.ps, assd_dist * self.ps
def compute_hausdorff_dist(modelresults_folder,resultsdir_3Dlabels,ROI_list, image_identifier_3D): # create files to save the hausdorff distances HD_header ='Patient\t' for ROI in ROI_list: HD_header = HD_header +f"{ROI}\t" HD_header=HD_header+"Mean\n" #num_ROIs = len(ROI_list) #with open(dice_file_2D, 'w') as f: #f.write(dicescores_header) #meandice_2D = 0 num_patients = 0 HD_file_3D = f'{modelresults_folder}/HD_3Dmodel.txt' with open(HD_file_3D, 'w') as f: f.write(HD_header) meanHD_3D = 0 HD_count = 0 # HD score vector: # column 1 = vessie # column 2 = rectum # column 3 = prostate simu_HD_vector = np.array(([],[],[])) clb_HD_vector = np.array(([],[],[])) rennes_HD_vector = np.array(([],[],[])) pseudoCT_HD_vector = np.array(([],[],[])) simu_count_invalids = np.zeros(3) clb_count_invalids = np.zeros(3) rennes_count_invalids = np.zeros(3) pseudoCT_count_invalids = np.zeros(3) #simu_HD_vector = np.zeros([simu_testimages_range[1]-simu_testimages_range[0]+1, 3]) #clb_HD_vector = np.zeros([clb_testimages_range[1]-clb_testimages_range[0]+1, 3]) #rennes_HD_vector = np.zeros([rennes_testimages_range[1]-rennes_testimages_range[0]+1, 3]) if '1mm' in resultsdir_3Dlabels: spacing = 1 else: spacing = 2 print(spacing) # calculate HD between predicted and real labels 3D for reallabel in sorted(glob.glob(f'{resultsdir_3Dlabels}/*.nii*')): #print(rennes_HD_vector) patient_id = int(os.path.basename(reallabel).replace(image_identifier_3D+'_','').replace('.nii.gz','')) num_patients=num_patients+1 #label_2Dmodel = reallabel.replace(f'{resultsdir_3Dlabels}/segmap',f'{modelresults_folder}/segm_results/merged_labels/label_patient') reallabel_array=itk.GetArrayFromImage(itk.imread(reallabel)) #label_2Dmodel_array=itk.GetArrayFromImage(itk.imread(label_2Dmodel)) #HDscores_2D = '' #meanHD_2D_perpatient = 0 label_3Dmodel = reallabel.replace(f'{resultsdir_3Dlabels}',f'{modelresults_folder}/OUTPUT_DIRECTORY_3D') label_3Dmodel_array=itk.GetArrayFromImage(itk.imread(label_3Dmodel)) HD_3D_text = os.path.basename(reallabel) + '\t' #'' meanHD_3D_perpatient = 0 #num_labels_2D = 0 num_labels_3D = 0 patient_HD_row = np.zeros([1,3]) patient_count_invalids = np.zeros(3) for ROI_ind, ROI in enumerate(ROI_list): invalid = False # the class value = ROI_ind + 1 k=ROI_ind +1 true_points_reallabel = np.argwhere(reallabel_array==k) if true_points_reallabel.size == 0: invalid = True HD_3D = float('inf') print('invalid1') else: top_left = true_points_reallabel.min(axis=0) bottom_right = true_points_reallabel.max(axis=0) label_3Dmodel_array_cropped = label_3Dmodel_array[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1] reallabel_array_cropped = reallabel_array[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1] true_points_pred = np.argwhere(label_3Dmodel_array_cropped==k) if ( true_points_pred.size == 0 ): invalid = True HD_3D = float('inf') print('invalid2') else: # only for rectum, only calculate the dice in the slices where there is a prediction (because it is not fully delineated in the real labels) if "rect" in ROI: top_left = true_points_pred.min(axis=0) bottom_right = true_points_pred.max(axis=0) label_3Dmodel_array_cropped = label_3Dmodel_array_cropped[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1] reallabel_array_cropped = reallabel_array_cropped[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1] true_points_pred = np.argwhere(label_3Dmodel_array_cropped==k) true_points_real = np.argwhere(reallabel_array_cropped==k) if ( true_points_pred.size == 0 ) or (true_points_real.size==0): invalid = True HD_3D = float('inf') print('invalid2') else: HD_3D = hd95(np.where(label_3Dmodel_array_cropped==k,1,0),np.where(reallabel_array_cropped==k,1,0),spacing,True) else: HD_3D = hd95(np.where(label_3Dmodel_array_cropped==k,1,0),np.where(reallabel_array_cropped==k,1,0),spacing,True) # find HD ROI index if "vessie" in ROI: HD_ROI_ind = 0 elif "rect" in ROI: HD_ROI_ind = 1 elif "prostate" in ROI: HD_ROI_ind = 2 else: HD_ROI_ind = -1 if (HD_ROI_ind != -1): patient_HD_row[0,HD_ROI_ind] = HD_3D #if (patient_id >= simu_testimages_range[0]) & (patient_id <= simu_testimages_range[1]): # simu_HD_vector[patient_id - simu_testimages_range[0],HD_ROI_ind] = HD_3D #elif (patient_id >= clb_testimages_range[0]) & (patient_id <= clb_testimages_range[1]): # clb_HD_vector[patient_id - clb_testimages_range[0],HD_ROI_ind] = HD_3D #elif (patient_id >= rennes_testimages_range[0]) & (patient_id <= rennes_testimages_range[1]): # rennes_HD_vector[patient_id - rennes_testimages_range[0],HD_ROI_ind] = HD_3D HD_3D_text=HD_3D_text+f'{HD_3D}\t' #print(np.sum(label_3Dmodel_array[reallabel_array==k]==k)*2.0 ) #print(np.sum(label_3Dmodel_array[label_3Dmodel_array==k]==k) ) #print( np.sum(reallabel_array[reallabel_array==k]==k)) #print(HD_3D) if not (invalid or math.isnan(HD_3D) or (HD_3D == float('inf'))): meanHD_3D_perpatient = meanHD_3D_perpatient + HD_3D num_labels_3D = num_labels_3D + 1 elif HD_ROI_ind != -1: patient_count_invalids[HD_ROI_ind] = patient_count_invalids[HD_ROI_ind] + 1 if (patient_id >= simu_testimages_range[0]) & (patient_id <= simu_testimages_range[1]): if np.size(simu_HD_vector) == 0: simu_HD_vector = patient_HD_row else: simu_HD_vector = np.concatenate((simu_HD_vector,patient_HD_row),axis=0) simu_count_invalids = simu_count_invalids + patient_count_invalids #simu_HD_vector[patient_id - simu_testimages_range[0],HD_ROI_ind] = HD_3D elif (patient_id >= clb_testimages_range[0]) & (patient_id <= clb_testimages_range[1]): if np.size(clb_HD_vector) == 0: clb_HD_vector = patient_HD_row else: clb_HD_vector = np.concatenate((clb_HD_vector,patient_HD_row),axis=0) clb_count_invalids = clb_count_invalids + patient_count_invalids #clb_HD_vector[patient_id - clb_testimages_range[0],HD_ROI_ind] = HD_3D elif (patient_id >= rennes_testimages_range[0]) & (patient_id <= rennes_testimages_range[1]): if np.size(rennes_HD_vector) == 0: rennes_HD_vector = patient_HD_row else: rennes_HD_vector = np.concatenate((rennes_HD_vector,patient_HD_row),axis=0) rennes_count_invalids = rennes_count_invalids + patient_count_invalids #rennes_HD_vector[patient_id - rennes_testimages_range[0],HD_ROI_ind] = HD_3D elif (patient_id >= pseudoCT_testimages_range[0]) & (patient_id <= pseudoCT_testimages_range[1]): if np.size(pseudoCT_HD_vector) == 0: pseudoCT_HD_vector = patient_HD_row else: pseudoCT_HD_vector = np.concatenate((pseudoCT_HD_vector,patient_HD_row),axis=0) pseudoCT_count_invalids = pseudoCT_count_invalids + patient_count_invalids #meanHD_2D_perpatient=meanHD_2D_perpatient/num_labels_2D #meanHD_2D = meanHD_2D+meanHD_2D_perpatient #HDscores_2D=HDscores_2D+f"{meanHD_2D_perpatient}\n" #with open(HD_file_2D, "a") as f: # f.write(HDscores_2D) if num_labels_3D==0: HD_3D_text=HD_3D_text+"nan\n" else: meanHD_3D_perpatient=meanHD_3D_perpatient/num_labels_3D meanHD_3D = meanHD_3D+meanHD_3D_perpatient HD_count = HD_count +1 HD_3D_text=HD_3D_text+f"{meanHD_3D_perpatient}\n" with open(HD_file_3D, "a") as f: f.write(HD_3D_text) np.savetxt(f'{modelresults_folder}/simu_HD_vector.csv', simu_HD_vector, delimiter=',') np.savetxt(f'{modelresults_folder}/clb_HD_vector.csv', clb_HD_vector, delimiter=',') np.savetxt(f'{modelresults_folder}/rennes_HD_vector.csv', rennes_HD_vector, delimiter=',') np.savetxt(f'{modelresults_folder}/pseudoCT_HD_vector.csv', pseudoCT_HD_vector, delimiter=',') np.savetxt(f'{modelresults_folder}/simu_invalids_count_hd.csv', simu_count_invalids, delimiter=',') np.savetxt(f'{modelresults_folder}/clb_invalids_count_hd.csv', clb_count_invalids, delimiter=',') np.savetxt(f'{modelresults_folder}/rennes_invalids_count_hd.csv', rennes_count_invalids, delimiter=',') np.savetxt(f'{modelresults_folder}/pseudoCT_invalids_count_hd.csv', pseudoCT_count_invalids, delimiter=',') boxplot_array_vessie, boxplot_array_rect, boxplot_array_prostate, boxplot_labels, bladderboxplot_invalids_vector, rectumboxplot_invalids_vector, prostateboxplot_invalids_vector = build_boxplot_data(simu_HD_vector,clb_HD_vector,rennes_HD_vector,pseudoCT_HD_vector, simu_count_invalids, clb_count_invalids, rennes_count_invalids, pseudoCT_count_invalids) plot_boxplot(boxplot_array_vessie,boxplot_labels, bladderboxplot_invalids_vector, f'Bladder HD scores (in mm) for test dataset', f'{modelresults_folder}/vessie_hausdorff.png') plot_boxplot(boxplot_array_rect,boxplot_labels, rectumboxplot_invalids_vector, f'Rectum HD scores (in mm) for test dataset', f'{modelresults_folder}/rectum_hausdorff.png') plot_boxplot(boxplot_array_prostate,boxplot_labels, prostateboxplot_invalids_vector, f'Prostate HD scores (in mm) for test dataset', f'{modelresults_folder}/prostate_hausdorff.png') if 'CBCT' in modelresults_folder: CBCT_simu_HD_vector = simu_HD_vector CBCT_clb_HD_vector = clb_HD_vector CBCT_rennes_HD_vector = rennes_HD_vector # try to find the HD score vectors for the similar model trained with CBCT CT_folder = modelresults_folder.replace('CBCT','CT') if os.path.isfile(f'{CT_folder}/simu_HD_vector.csv'): CT_simu_HD_vector = np.loadtxt(f'{CT_folder}/simu_HD_vector.csv', delimiter=',') if os.path.isfile(f'{CT_folder}/clb_HD_vector.csv'): CT_clb_HD_vector = np.loadtxt(f'{CT_folder}/clb_HD_vector.csv', delimiter=',') if os.path.isfile(f'{CT_folder}/rennes_HD_vector.csv'): CT_rennes_HD_vector = np.loadtxt(f'{CT_folder}/rennes_HD_vector.csv', delimiter=',') CBCT_simu_invalids_vector = simu_count_invalids CBCT_clb_invalids_vector = clb_count_invalids CBCT_rennes_invalids_vector = rennes_count_invalids # try to find the invalid vectors for the similar model trained with CBCT CT_folder = modelresults_folder.replace('CBCT','CT') if os.path.isfile(f'{CT_folder}/simu_invalids_count_hd.csv'): CT_simu_invalids_vector = np.loadtxt(f'{CT_folder}/simu_invalids_count_hd.csv', delimiter=',') if os.path.isfile(f'{CT_folder}/clb_invalids_count_hd.csv'): CT_clb_invalids_vector = np.loadtxt(f'{CT_folder}/clb_invalids_count_hd.csv', delimiter=',') if os.path.isfile(f'{CT_folder}/rennes_invalids_count_hd.csv'): CT_rennes_invalids_vector = np.loadtxt(f'{CT_folder}/rennes_invalids_count_hd.csv', delimiter=',') elif 'CT' in modelresults_folder: # try to find the HD score vectors for the similar model trained with CBCT #print(simu_HD_vector) CT_simu_HD_vector = simu_HD_vector CT_clb_HD_vector = clb_HD_vector CT_rennes_HD_vector = rennes_HD_vector CBCT_folder = modelresults_folder.replace('CT','CBCT') if os.path.isfile(f'{CBCT_folder}/simu_HD_vector.csv'): CBCT_simu_HD_vector = np.loadtxt(f'{CBCT_folder}/simu_HD_vector.csv', delimiter=',') if os.path.isfile(f'{CBCT_folder}/clb_HD_vector.csv'): CBCT_clb_HD_vector = np.loadtxt(f'{CBCT_folder}/clb_HD_vector.csv', delimiter=',') if os.path.isfile(f'{CBCT_folder}/rennes_HD_vector.csv'): CBCT_rennes_HD_vector = np.loadtxt(f'{CBCT_folder}/rennes_HD_vector.csv', delimiter=',') CT_simu_invalids_vector = simu_count_invalids CT_clb_invalids_vector = clb_count_invalids CT_rennes_invalids_vector = rennes_count_invalids # try to find the invalid vectors for the similar model trained with CBCT CBCT_folder = modelresults_folder.replace('CT','CBCT') if os.path.isfile(f'{CBCT_folder}/simu_invalids_count_hd.csv'): CBCT_simu_invalids_vector = np.loadtxt(f'{CBCT_folder}/simu_invalids_count_hd.csv', delimiter=',') if os.path.isfile(f'{CBCT_folder}/clb_invalids_count_hd.csv'): CBCT_clb_invalids_vector = np.loadtxt(f'{CBCT_folder}/clb_invalids_count_hd.csv', delimiter=',') if os.path.isfile(f'{CBCT_folder}/rennes_invalids_count_hd.csv'): CBCT_rennes_invalids_vector = np.loadtxt(f'{CBCT_folder}/rennes_invalids_count_hd.csv', delimiter=',') try: # simu cbct plot_boxplot([CBCT_simu_HD_vector[:,0], CT_simu_HD_vector[:,0]],['Trained with CBCT','Trained with CT'], [CBCT_simu_invalids_vector[0], CT_simu_invalids_vector[0]] , f'Bladder HD scores (in mm) for simulated cbct test dataset', f'{modelresults_folder}/simu_CTvsCBCT_vessie_hausdorff.png') plot_boxplot([CBCT_simu_HD_vector[:,1], CT_simu_HD_vector[:,1]],['Trained with CBCT','Trained with CT'], [CBCT_simu_invalids_vector[1], CT_simu_invalids_vector[1]], f'Rectum HD scores (in mm) for simulated cbct test dataset', f'{modelresults_folder}/simu_CTvsCBCT_rectum_hausdorff.png') plot_boxplot([CBCT_simu_HD_vector[:,2], CT_simu_HD_vector[:,2]],['Trained with CBCT','Trained with CT'], [CBCT_simu_invalids_vector[2], CT_simu_invalids_vector[2]], f'Prostate HD scores (in mm) for simulated cbct test dataset', f'{modelresults_folder}/simu_CTvsCBCT_prostate_hausdorff.png') except: print('Simu HD scores not available') try: # CLB cbct plot_boxplot([CBCT_clb_HD_vector[:,0], CT_clb_HD_vector[:,0]],['Trained with CBCT','Trained with CT'],[CBCT_clb_invalids_vector[0], CT_clb_invalids_vector[0]], f'Bladder HD scores (in mm) for Lyon cbct test dataset', f'{modelresults_folder}/clb_CTvsCBCT_vessie_hausdorff.png') plot_boxplot([CBCT_clb_HD_vector[:,1], CT_clb_HD_vector[:,1]],['Trained with CBCT','Trained with CT'],[CBCT_clb_invalids_vector[1], CT_clb_invalids_vector[1]], f'Rectum HD scores (in mm) for Lyon cbct test dataset', f'{modelresults_folder}/clb_CTvsCBCT_rectum_hausdorff.png') plot_boxplot([CBCT_clb_HD_vector[:,2], CT_clb_HD_vector[:,2]],['Trained with CBCT','Trained with CT'],[CBCT_clb_invalids_vector[2], CT_clb_invalids_vector[2]], f'Prostate HD scores (in mm) for Lyon cbct test dataset', f'{modelresults_folder}/clb_CTvsCBCT_prostate_hausdorff.png') except: print('CLB HD scores not available') try: # Rennes cbct plot_boxplot([CBCT_rennes_HD_vector[:,0], CT_rennes_HD_vector[:,0]],['Trained with CBCT','Trained with CT'], [CBCT_rennes_invalids_vector[0], CT_rennes_invalids_vector[0]], f'Bladder HD scores (in mm) for Rennes cbct test dataset', f'{modelresults_folder}/rennes_CTvsCBCT_vessie_hausdorff.png') plot_boxplot([CBCT_rennes_HD_vector[:,1], CT_rennes_HD_vector[:,1]],['Trained with CBCT','Trained with CT'], [CBCT_rennes_invalids_vector[1], CT_rennes_invalids_vector[1]], f'Rectum HD scores (in mm) for Rennes cbct test dataset', f'{modelresults_folder}/rennes_CTvsCBCT_rectum_hausdorff.png') plot_boxplot([CBCT_rennes_HD_vector[:,2], CT_rennes_HD_vector[:,2]],['Trained with CBCT','Trained with CT'], [CBCT_rennes_invalids_vector[2], CT_rennes_invalids_vector[2]], f'Prostate HD scores (in mm) for Rennes cbct test dataset', f'{modelresults_folder}/rennes_CTvsCBCT_prostate_hausdorff.png') except: print('Rennes HD scores not available') # one box plot for all training and test datasets try: #plot_boxplot([CBCT_simu_HD_vector[:,0], CT_simu_HD_vector[:,0], CBCT_clb_HD_vector[:,0], CT_clb_HD_vector[:,0], CBCT_rennes_HD_vector[:,0], CT_rennes_HD_vector[:,0]],['simu CBCT\nsimu CBCT','real CT\nsimu CBCT','simu CBCT\nCLB CBCT','real CT\nCLB CBCT','simu CBCT\nCEM CBCT','real CT\nCEM CBCT'], f'Vessie HD scores for all cbct test subsets', f'{modelresults_folder}/CTvsCBCT_vessie_hausdorff.png') plot_boxplot([CBCT_simu_HD_vector[:,0], CT_simu_HD_vector[:,0], CBCT_clb_HD_vector[:,0], CT_clb_HD_vector[:,0], CBCT_rennes_HD_vector[:,0], CT_rennes_HD_vector[:,0]],['simu CBCT','real CT','simu CBCT','real CT','simu CBCT','real CT'], [CBCT_simu_invalids_vector[0], CT_simu_invalids_vector[0],CBCT_clb_invalids_vector[0], CT_clb_invalids_vector[0],CBCT_rennes_invalids_vector[0], CT_rennes_invalids_vector[0]], f'Bladder HD scores (in mm) for all test CBCT subsets', f'{modelresults_folder}/CTvsCBCT_vessie_hausdorff.png') plot_boxplot([CBCT_simu_HD_vector[:,1], CT_simu_HD_vector[:,1], CBCT_clb_HD_vector[:,1], CT_clb_HD_vector[:,1], CBCT_rennes_HD_vector[:,1], CT_rennes_HD_vector[:,1]],['simu CBCT','real CT','simu CBCT','real CT','simu CBCT','real CT'], [CBCT_simu_invalids_vector[1], CT_simu_invalids_vector[1],CBCT_clb_invalids_vector[1], CT_clb_invalids_vector[1],CBCT_rennes_invalids_vector[1], CT_rennes_invalids_vector[1]], f'Rectum HD scores (in mm) for all test CBCT subsets', f'{modelresults_folder}/CTvsCBCT_rectum_hausdorff.png') plot_boxplot([CBCT_simu_HD_vector[:,2], CT_simu_HD_vector[:,2], CBCT_clb_HD_vector[:,2], CT_clb_HD_vector[:,2], CBCT_rennes_HD_vector[:,2], CT_rennes_HD_vector[:,2]],['simu CBCT','real CT','simu CBCT','real CT','simu CBCT','real CT'], [CBCT_simu_invalids_vector[2], CT_simu_invalids_vector[2],CBCT_clb_invalids_vector[2], CT_clb_invalids_vector[2],CBCT_rennes_invalids_vector[2], CT_rennes_invalids_vector[2]], f'Prostate HD scores (in mm) for all test CBCT subsets', f'{modelresults_folder}/CTvsCBCT_prostate_hausdorff.png') except: print('HD scores not available') # calculate mean of means #with open(HD_file_2D, "a") as f: # f.write(f"Total mean: {meanHD_2D/num_patients}\n") with open(HD_file_3D, "a") as f: f.write(f"Total mean: {meanHD_3D/HD_count}\n")
import os from medpy.metric.binary import hd95 import SimpleITK as sitk import pkg_resources if __name__ == '__main__': parser = argparse.ArgumentParser(prog='Hausdorff95', formatter_class=argparse.RawTextHelpFormatter, description='\nThis code is used to get the Hausdorff 95th percentile. '+\ 'For questions and feedback contact: [email protected]') parser.add_argument('-gt', dest='groundTruth', type=str, help='The ground truth image for comparison.\n', required=True) parser.add_argument('-m', dest='maskImage', type=str, help='The annotated mask to compare against ground truth.\n', required=True) # parser.add_argument('-v', '--version', action='version', # version=pkg_resources.require("Hausdorff95")[0].version, help="Show program's version number and exit.") # disabled because pyinstaller doesn't import it properly args = parser.parse_args() groundTruth = os.path.abspath(args.groundTruth) maskImage = os.path.abspath(args.maskImage) gt = sitk.GetArrayFromImage(sitk.ReadImage(groundTruth)) mk = sitk.GetArrayFromImage(sitk.ReadImage(maskImage)) print(hd95(gt,mk))
# list of output segmentations list_of_output = ['', '', '', ......] all_assd = [] all_hd95 = [] for x in range(len(list_of_gt)): gt_label = '/path/to/current/groundtruth/segmentation.nii.gz' gt_path = read_nifti(gt_label) read_segm_output = read_nifti(segm_output) segm_output_cc = getLargestCC(read_segm_output) segm_output_cc_path = '/path/to/current/segmentation/output.nii.gz' save_nifti(segm_output_cc, segm_output_cc_path) result, h1 = load(segm_output_cc_path) reference, h2 = load(gt_path) all_hd95.append(binary.hd95(reference, result, voxelspacing =voxelspacing)) all_assd.append(binary.assd(reference, result, voxelspacing =voxelspacing)) print(">>>>>>>") print("hd95 for all subjects", np.mean(all_hd95), np.std(all_hd95)) print(">>>>>>>") print("assd for all subjects", np.mean(all_assd), np.std(all_assd)) print(">>>>>>>") print("asd for all subjects", np.mean(all_asd), np.std(all_asd))