def update_eval_metrics(preds, labels, eval_metrics):
if len(labels.shape) == 2:
preds = np.expand_dims(preds, axis=0)
labels = np.expand_dims(labels, axis=0)
N = labels.shape[0]
for i in range(N):
pred = preds[i, :, :]
label = labels[i, :, :]
eval_metrics['dice score'].append(dc(pred, label))
eval_metrics['precision'].append(precision(pred, label))
eval_metrics['recall'].append(recall(pred, label))
eval_metrics['sensitivity'].append(sensitivity(pred, label))
eval_metrics['specificity'].append(specificity(pred, label))
if np.sum(pred) > 0 and np.sum(label) > 0:
eval_metrics['hausdorff'].append(hd(pred, label))
eval_metrics['hausdorff 95%'].append(hd95(pred, label))
eval_metrics['asd'].append(asd(pred, label))
eval_metrics['assd'].append(assd(pred, label))
eval_metrics['jaccard'].append(jc(pred, label))
else:
eval_metrics['hausdorff'].append('nan')
eval_metrics['hausdorff 95%'].append('nan')
eval_metrics['asd'].append('nan')
eval_metrics['assd'].append('nan')
eval_metrics['jaccard'].append('nan')
return eval_metrics
def __compute_subcortical_structures_location(self, volume, spacing, category=None, reference='BCB'):
distances = {}
overlaps = {}
distances_columns = []
overlaps_columns = []
tract_cutoff = 0.5
if reference == 'BrainLab':
tract_cutoff = 0.25
tracts_dict = ResourcesConfiguration.getInstance().subcortical_structures['MNI'][reference]
for i, tfn in enumerate(tracts_dict.keys()):
reg_tract_ni = nib.load(tracts_dict[tfn])
reg_tract = reg_tract_ni.get_data()[:]
reg_tract[reg_tract < tract_cutoff] = 0
reg_tract[reg_tract >= tract_cutoff] = 1
overlap_volume = np.logical_and(reg_tract, volume).astype('uint8')
distances_columns.append('distance_' + tfn.split('.')[0][:-4] + '_' + category)
overlaps_columns.append('overlap_' + tfn.split('.')[0][:-4] + '_' + category)
if np.count_nonzero(overlap_volume) != 0:
distances[tfn] = -1.
overlaps[tfn] = (np.count_nonzero(overlap_volume) / np.count_nonzero(volume)) * 100.
else:
dist = -1.
if np.count_nonzero(reg_tract) > 0:
dist = hd95(volume, reg_tract, voxelspacing=reg_tract_ni.header.get_zooms(), connectivity=1)
distances[tfn] = dist
overlaps[tfn] = 0.
self.diagnosis_parameters.statistics[category]['Overall'].mni_space_subcortical_structures_overlap[reference] = overlaps
self.diagnosis_parameters.statistics[category]['Overall'].mni_space_subcortical_structures_distance[reference] = distances
def hd95_multiply(logits, targets, cls_id=None):
if isinstance(logits, torch.Tensor):
logits = logits.cpu().numpy()
targets = targets.cpu().numpy()
batch_size, class_cnt = logits.shape[0], logits.shape[1]
#print('batch_size, class_cnt ', batch_size, class_cnt )
hd95_score = []
for class_index in range(class_cnt):
#print(class_index, np.unique(logits.argmax(axis=1)))
predict = logits.argmax(axis=-3) == class_index
target = (targets == class_index)
#print('predict, target', class_index, predict.shape, target.shape,
#predict.sum(), target.sum(), np.unique(targets))
score = [
hd95(res, ref) for res, ref in zip(predict, target)
if np.count_nonzero(res) > 0 and np.count_nonzero(ref) > 0
]
#print('class_index=', class_index, score)
hd95_score.append(np.mean(score))
#print('hd95_score', hd95_score)
if cls_id is None:
return torch.Tensor(hd95_score)
else:
return torch.Tensor(hd95_score)[cls_id]
def Hausdorff(pred, gt):
batch_s = pred.size(0)
HDC = np.zeros(batch_s)
pred[pred > 0] = 1
pred = pred.detach().cpu().numpy()
gt[gt > 0] = 1
gt = gt.detach().cpu().numpy()
for jj in range(batch_s):
if pred[jj, :, :].sum() > 0 and gt[jj, :, :].sum() > 0:
HDC[jj] = binary.hd95(pred[jj, :, :], gt[jj, :, :])
else:
HDC[jj] = 0
return HDC
def __compute_multifocality(self, volume, spacing):
multifocality = None
multifocal_elements = None
multifocal_largest_minimum_distance = None
tumor_clusters = measurements.label(volume)[0]
tumor_clusters_labels = regionprops(tumor_clusters)
if len(tumor_clusters_labels) > 1:
multifocality = False
multifocal_elements = 0
# Computing the radius of the largest component.
radiuses = []
parts_labels = []
for l in range(len(tumor_clusters_labels)):
volume_ml = np.count_nonzero(tumor_clusters == (l + 1)) * np.prod(spacing[0:3]) * 1e-3
if volume_ml >= 0.1: # Discarding tumor parts smaller than 0.1 ml
multifocal_elements = multifocal_elements + 1
radiuses.append(tumor_clusters_labels[l].equivalent_diameter / 2.)
parts_labels.append(l)
max_radius = np.max(radiuses)
max_radius_index = parts_labels[radiuses.index(max_radius)]
# Computing the minimum distances between each foci
main_tumor_label = np.zeros(volume.shape)
main_tumor_label[tumor_clusters == (max_radius_index + 1)] = 1
for l, lab in enumerate(parts_labels):
if lab != max_radius_index:
satellite_label = np.zeros(volume.shape)
satellite_label[tumor_clusters == (lab + 1)] = 1
dist = hd95(satellite_label, main_tumor_label, voxelspacing=spacing, connectivity=1)
if multifocal_largest_minimum_distance is None:
multifocal_largest_minimum_distance = dist
elif dist > multifocal_largest_minimum_distance:
multifocal_largest_minimum_distance = dist
if multifocal_largest_minimum_distance >= 5.0:
multifocality = True
else:
multifocality = False
multifocal_elements = 1
multifocal_largest_minimum_distance = -1.
self.diagnosis_parameters.tumor_multifocal = multifocality
self.diagnosis_parameters.tumor_parts = multifocal_elements
self.diagnosis_parameters.tumor_multifocal_distance = multifocal_largest_minimum_distance
def run_hd(image1, image2, mode='max'):
image1_data, hd1 = nrrd.read(image1)
space_x = np.abs(hd1['space directions'][0, 0])
space_y = np.abs(hd1['space directions'][1, 1])
space_z = np.abs(hd1['space directions'][2, 2])
image2_data, hd2 = nrrd.read(image2)
if (hd1['space origin'] == hd2['space origin']).all():
if mode == 'max':
hd_val = hd(image1_data, image2_data, (space_x, space_y, space_z))
elif mode == '95':
hd_val = hd95(image1_data, image2_data,
(space_x, space_y, space_z))
else:
raise Exception(
'Unknown mode "{}". Possible values are "max" or "95".'.format(
mode))
else:
hd_val = None
return hd_val
def channelwise_hd95(pred, target):
""" calculate channel-wise 95 percentile symmetric Hausdorff distance,
especially for pred with multiple channels.
:param pred: ndarray with size [C, H, W], predicted boundary
:param target: ndarray with size [C, H, W], GT boundary
:return mean_hd95: float, average hdf
"""
max_hd95 = 2 * target.shape[1]
channel_res = []
for c_inx in range(0, len(pred)):
if np.sum(target[c_inx]) != 0:
if np.sum(pred[c_inx]) == 0:
res = max_hd95
else:
res = hd95(pred[c_inx], target[c_inx])
channel_res.append(res)
mean_hd95 = sum(channel_res) / len(channel_res)
return mean_hd95
def hausdorff95(prediction: torch.Tensor, target: torch.Tensor,
merge_operation: Callable[
[torch.Tensor], float] = torch.max) -> torch.Tensor:
"""
95th percentile of the Hausdorff Distance.
Computes the 95th percentile of the (symmetric) Hausdorff Distance (HD)
between the binary objects in two images.
Heavily depends on medpy.metric.binary.hd95, so check it out for more
details.
Args:
prediction: Network output.
Dimensions - (Batch, Class, Depth, Height, Width)
target: Target values.
Dimensions - (Batch, Class, Depth, Height, Width)
merge_operation: Operation to merge results from all the slices in
the batch into a single value. Should be a function that accepts
two dimensional tensors (Batch, Depth).
Good examples are: torch.mean, torch.max, torch.min.
Returns:
torch.Tensor: Hausdorff distance for the provided volume
"""
assert utils.is_binary(prediction), "Predictions must be binary"
assert utils.is_binary(target), "Target must be binary"
prediction = prediction.cpu().detach().numpy()
target = target.cpu().detach().numpy()
volumes_count, _, slices_count, _, _ = prediction.shape
results = np.zeros((volumes_count, slices_count))
for volume_idx in range(volumes_count):
for slice_idx in range(slices_count):
prediction_slice = prediction[
volume_idx, FIRST_CLASS, slice_idx, ...]
target_slice = target[volume_idx, FIRST_CLASS, slice_idx, ...]
if utils.has_only_zeros(prediction_slice) or \
utils.has_only_zeros(target_slice):
results[volume_idx, slice_idx] = 0
else:
results[volume_idx, slice_idx] = mp.hd95(prediction_slice,
target_slice)
return merge_operation(torch.from_numpy(results))
def slicewise_hd95(pred, target, n_classes=3):
""" calculate Average Hausdorff distance between pred and target of single image
:param pred: ndarray with size [H, W], predicted bound
:param target: ndarray with size [H, W], ground truth
:param n_classes: int, # of classes
"""
max_hd95 = 2 * target.shape[0]
slice_res = []
for c_inx in range(1, n_classes):
pred_cinx = (pred == c_inx)
target_cinx = (target == c_inx)
if np.sum(target_cinx) != 0:
if np.sum(pred_cinx) == 0:
res = max_hd95
else:
res = hd95(pred_cinx, target_cinx)
slice_res.append(res)
mean_res = sum(slice_res) / len(slice_res)
return mean_res
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model-file', type=str, default='./logs/test1/20190506_221021.177567/checkpoint_200.pth.tar', help='Model path')
parser.add_argument('--datasetTest', type=list, default=[1], help='test folder id contain images ROIs to test')
parser.add_argument('--dataset', type=str, default='test', help='test folder id contain images ROIs to test')
parser.add_argument('-g', '--gpu', type=int, default=0)
parser.add_argument('--data-dir', default='../../../../Dataset/Fundus/', help='data root path')
parser.add_argument('--out-stride', type=int, default=16, help='out-stride of deeplabv3+',)
parser.add_argument('--sync-bn', type=bool, default=False, help='sync-bn in deeplabv3+')
parser.add_argument('--freeze-bn', type=bool, default=False, help='freeze batch normalization of deeplabv3+')
parser.add_argument('--movingbn', type=bool, default=False, help='moving batch normalization of deeplabv3+ in the test phase',)
parser.add_argument('--test-prediction-save-path', type=str, default='./results/rebuttle-0401/', help='Path root for test image and mask')
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu)
model_file = args.model_file
output_path = os.path.join(args.test_prediction_save_path, 'test' + str(args.datasetTest[0]), args.model_file.split('/')[-2])
# 1. dataset
composed_transforms_test = transforms.Compose([
tr.Normalize_tf(),
tr.ToTensor()
])
db_test = DL.FundusSegmentation(base_dir=args.data_dir, phase='test', splitid=args.datasetTest,
transform=composed_transforms_test, state='prediction')
batch_size = 12
test_loader = DataLoader(db_test, batch_size=batch_size, shuffle=False, num_workers=1)
# 2. model
model = DeepLab(num_classes=2, backbone='mobilenet', output_stride=args.out_stride,
sync_bn=args.sync_bn, freeze_bn=args.freeze_bn).cuda()
if torch.cuda.is_available():
model = model.cuda()
print('==> Loading %s model file: %s' %
(model.__class__.__name__, model_file))
# model_data = torch.load(model_file)
checkpoint = torch.load(model_file)
pretrained_dict = checkpoint['model_state_dict']
model_dict = model.state_dict()
# 1. filter out unnecessary keys
pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
# 2. overwrite entries in the existing state dict
model_dict.update(pretrained_dict)
# 3. load the new state dict
model.load_state_dict(model_dict)
if args.movingbn:
model.train()
else:
model.eval()
val_cup_dice = 0.0
val_disc_dice = 0.0
total_hd_OC = 0.0
total_hd_OD = 0.0
total_asd_OC = 0.0
total_asd_OD = 0.0
timestamp_start = datetime.now(pytz.timezone('Asia/Hong_Kong'))
total_num = 0
OC = []
OD = []
for batch_idx, (sample) in tqdm.tqdm(enumerate(test_loader),total=len(test_loader),ncols=80, leave=False):
data = sample['image']
target = sample['label']
img_name = sample['img_name']
if torch.cuda.is_available():
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
prediction, dc, sel, _ = model(data)
prediction = torch.nn.functional.interpolate(prediction, size=(target.size()[2], target.size()[3]), mode="bilinear")
data = torch.nn.functional.interpolate(data, size=(target.size()[2], target.size()[3]), mode="bilinear")
target_numpy = target.data.cpu()
imgs = data.data.cpu()
hd_OC = 100
asd_OC = 100
hd_OD = 100
asd_OD = 100
for i in range(prediction.shape[0]):
prediction_post = postprocessing(prediction[i], dataset=args.dataset)
cup_dice, disc_dice = dice_coeff_2label(prediction_post, target[i])
OC.append(cup_dice)
OD.append(disc_dice)
if np.sum(prediction_post[0, ...]) < 1e-4:
hd_OC = 100
asd_OC = 100
else:
hd_OC = binary.hd95(np.asarray(prediction_post[0, ...], dtype=np.bool),
np.asarray(target_numpy[i, 0, ...], dtype=np.bool))
asd_OC = binary.asd(np.asarray(prediction_post[0, ...], dtype=np.bool),
np.asarray(target_numpy[i, 0, ...], dtype=np.bool))
if np.sum(prediction_post[0, ...]) < 1e-4:
hd_OD = 100
asd_OD = 100
else:
hd_OD = binary.hd95(np.asarray(prediction_post[1, ...], dtype=np.bool),
np.asarray(target_numpy[i, 1, ...], dtype=np.bool))
asd_OD = binary.asd(np.asarray(prediction_post[1, ...], dtype=np.bool),
np.asarray(target_numpy[i, 1, ...], dtype=np.bool))
val_cup_dice += cup_dice
val_disc_dice += disc_dice
total_hd_OC += hd_OC
total_hd_OD += hd_OD
total_asd_OC += asd_OC
total_asd_OD += asd_OD
total_num += 1
for img, lt, lp in zip([imgs[i]], [target_numpy[i]], [prediction_post]):
img, lt = utils.untransform(img, lt)
save_per_img(img.numpy().transpose(1, 2, 0),
output_path,
img_name[i],
lp, lt, mask_path=None, ext="bmp")
print('OC:', OC)
print('OD:', OD)
import csv
with open('Dice_results.csv', 'a+') as result_file:
wr = csv.writer(result_file, dialect='excel')
for index in range(len(OC)):
wr.writerow([OC[index], OD[index]])
val_cup_dice /= total_num
val_disc_dice /= total_num
total_hd_OC /= total_num
total_asd_OC /= total_num
total_hd_OD /= total_num
total_asd_OD /= total_num
print('''\n==>val_cup_dice : {0}'''.format(val_cup_dice))
print('''\n==>val_disc_dice : {0}'''.format(val_disc_dice))
print('''\n==>average_hd_OC : {0}'''.format(total_hd_OC))
print('''\n==>average_hd_OD : {0}'''.format(total_hd_OD))
print('''\n==>ave_asd_OC : {0}'''.format(total_asd_OC))
print('''\n==>average_asd_OD : {0}'''.format(total_asd_OD))
with open(osp.join(output_path, '../test' + str(args.datasetTest[0]) + '_log.csv'), 'a') as f:
elapsed_time = (
datetime.now(pytz.timezone('Asia/Hong_Kong')) -
timestamp_start).total_seconds()
log = [['batch-size: '] + [batch_size] + [args.model_file] + ['cup dice coefficence: '] + \
[val_cup_dice] + ['disc dice coefficence: '] + \
[val_disc_dice] + ['average_hd_OC: '] + \
[total_hd_OC] + ['average_hd_OD: '] + \
[total_hd_OD] + ['ave_asd_OC: '] + \
[total_asd_OC] + ['average_asd_OD: '] + \
[total_asd_OD] + [elapsed_time]]
log = map(str, log)
f.write(','.join(log) + '\n')
def bench_pytorch_fp32():
### Load PyTorch model
pt_model = fetch_model(modelname="resunet",
num_channels=1,
num_classes=2,
num_filters=16)
checkpoint = torch.load(pytorch_model_path,
map_location=torch.device('cpu'))
pt_model.load_state_dict(checkpoint["model_state_dict"])
### Run PyTorch Inference
print(f"\n Starting PyTorch inference with {pytorch_model_path} ...")
_ = pt_model.eval()
pt_stats = []
with torch.no_grad():
for i, row in tqdm(dataset_df.iterrows()):
sub_id = row[sub_idx]
input_path = row[input_path_idx]
mask_path = row[mask_path_idx]
try:
mask_image = get_mask_image(mask_path)
input_image, patient_nib = get_input_image(input_path)
i_start = timer()
pt_output = pt_model(input_image)
i_end = timer()
p_start = timer()
pt_output = pt_output.cpu().numpy()[0][0]
pt_to_save = postprocess_output(pt_output, patient_nib.shape)
p_end = timer()
pt_dice_score = dice(pt_to_save, mask_image)
pt_hdorff_score = hd95(pt_to_save, mask_image)
pt_stat = [
i, sub_id, pt_hdorff_score, i_end - i_start,
p_end - p_start
]
pt_stats.append(pt_stat)
except Exception as e:
print(f" Inference Failed: {sub_id} ")
print(e)
print(f"Done PyTorch inference with {pytorch_model_path} ...")
pt_stats_df = pd.DataFrame(pt_stats)
date_time_str = datetime.now().strftime("%b-%d-%Y_%H-%M-%S")
csv_name = f"pt_stats_{date_time_str}.csv"
pt_stats_df.to_csv(csv_name, sep=',', header=False, index=False)
print(f"Saved {csv_name} ...")
print(
f"\n PyTorch Hausdorff Distance Mean: {pt_stats_df[:][2].mean():.5f}")
print(
f"PyTorch Total Inf Time: {pt_stats_df[:][3].sum():.2f} sec, Mean: {pt_stats_df[:][3].mean():.2f} sec"
)
return pt_stats_df
def bench_ov_int8():
# #### Load INT8 OpenVINO model
ov_model_dir = brainmage_root / 'openvino/int8_openvino_model'
modelname = "resunet_ma_int8"
model_xml = f'{ov_model_dir}/{modelname}.xml'
# Load network to the plugin
ie = IECore()
net = ie.read_network(model=model_xml)
exec_net = ie.load_network(network=net, device_name="CPU")
del net
input_layer = next(iter(exec_net.input_info))
output_layer = next(iter(exec_net.outputs))
# #### Run OpenVINO Inference
print(f"Starting OpenVINO inference with {ov_model_dir} ...")
ov_int8_stats = []
for i, row in tqdm(dataset_df.iterrows()):
sub_id = row[sub_idx]
input_path = row[input_path_idx]
mask_path = row[mask_path_idx]
try:
mask_image = get_mask_image(mask_path)
input_image, patient_nib = get_input_image(input_path)
i_start = timer()
ov_output = exec_net.infer(inputs={input_layer: input_image})
i_end = timer()
p_start = timer()
ov_output = ov_output[output_layer][0][0]
ov_to_save = postprocess_output(ov_output, patient_nib.shape)
p_end = timer()
ov_dice_score = dice(ov_to_save, mask_image)
ov_hdorff_score = hd95(ov_to_save, mask_image)
ov_stat = [
i, sub_id, ov_hdorff_score, i_end - i_start, p_end - p_start
]
ov_int8_stats.append(ov_stat)
except:
print(f" Inference Failed: {sub_id} ")
print(f"Done OpenVINO inference with {ov_model_dir} ...")
ov_int8_stats_df = pd.DataFrame(ov_int8_stats)
date_time_str = datetime.now().strftime("%b-%d-%Y_%H-%M-%S")
csv_name = f"ov_int8_stats_{date_time_str}.csv"
ov_int8_stats_df.to_csv(csv_name, sep=',', header=False, index=False)
print(f"Saved {csv_name} ...")
print(
f"\n OpenVINO INT8 Hausdorff Distance Mean: {ov_int8_stats_df[:][2].mean():.5f}"
)
print(
f"OpenVINO INT8 Total Inf Time: {ov_int8_stats_df[:][3].sum():.2f} sec, Mean: {ov_int8_stats_df[:][3].mean():.2f} sec"
)
return ov_int8_stats_df
def get_hd_assd(self, b):
hd95_dist = hd95(self.pred_mid_rl[b], self.gt_mid[b])
assd_dist = assd(self.pred_mid_rl[b], self.gt_mid[b])
return hd95_dist * self.ps, assd_dist * self.ps
def compute_hausdorff_dist(modelresults_folder,resultsdir_3Dlabels,ROI_list, image_identifier_3D):
# create files to save the hausdorff distances
HD_header ='Patient\t'
for ROI in ROI_list:
HD_header = HD_header +f"{ROI}\t"
HD_header=HD_header+"Mean\n"
#num_ROIs = len(ROI_list)
#with open(dice_file_2D, 'w') as f:
#f.write(dicescores_header)
#meandice_2D = 0
num_patients = 0
HD_file_3D = f'{modelresults_folder}/HD_3Dmodel.txt'
with open(HD_file_3D, 'w') as f:
f.write(HD_header)
meanHD_3D = 0
HD_count = 0
# HD score vector:
# column 1 = vessie
# column 2 = rectum
# column 3 = prostate
simu_HD_vector = np.array(([],[],[]))
clb_HD_vector = np.array(([],[],[]))
rennes_HD_vector = np.array(([],[],[]))
pseudoCT_HD_vector = np.array(([],[],[]))
simu_count_invalids = np.zeros(3)
clb_count_invalids = np.zeros(3)
rennes_count_invalids = np.zeros(3)
pseudoCT_count_invalids = np.zeros(3)
#simu_HD_vector = np.zeros([simu_testimages_range[1]-simu_testimages_range[0]+1, 3])
#clb_HD_vector = np.zeros([clb_testimages_range[1]-clb_testimages_range[0]+1, 3])
#rennes_HD_vector = np.zeros([rennes_testimages_range[1]-rennes_testimages_range[0]+1, 3])
if '1mm' in resultsdir_3Dlabels:
spacing = 1
else:
spacing = 2
print(spacing)
# calculate HD between predicted and real labels 3D
for reallabel in sorted(glob.glob(f'{resultsdir_3Dlabels}/*.nii*')):
#print(rennes_HD_vector)
patient_id = int(os.path.basename(reallabel).replace(image_identifier_3D+'_','').replace('.nii.gz',''))
num_patients=num_patients+1
#label_2Dmodel = reallabel.replace(f'{resultsdir_3Dlabels}/segmap',f'{modelresults_folder}/segm_results/merged_labels/label_patient')
reallabel_array=itk.GetArrayFromImage(itk.imread(reallabel))
#label_2Dmodel_array=itk.GetArrayFromImage(itk.imread(label_2Dmodel))
#HDscores_2D = ''
#meanHD_2D_perpatient = 0
label_3Dmodel = reallabel.replace(f'{resultsdir_3Dlabels}',f'{modelresults_folder}/OUTPUT_DIRECTORY_3D')
label_3Dmodel_array=itk.GetArrayFromImage(itk.imread(label_3Dmodel))
HD_3D_text = os.path.basename(reallabel) + '\t' #''
meanHD_3D_perpatient = 0
#num_labels_2D = 0
num_labels_3D = 0
patient_HD_row = np.zeros([1,3])
patient_count_invalids = np.zeros(3)
for ROI_ind, ROI in enumerate(ROI_list):
invalid = False
# the class value = ROI_ind + 1
k=ROI_ind +1
true_points_reallabel = np.argwhere(reallabel_array==k)
if true_points_reallabel.size == 0:
invalid = True
HD_3D = float('inf')
print('invalid1')
else:
top_left = true_points_reallabel.min(axis=0)
bottom_right = true_points_reallabel.max(axis=0)
label_3Dmodel_array_cropped = label_3Dmodel_array[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1]
reallabel_array_cropped = reallabel_array[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1]
true_points_pred = np.argwhere(label_3Dmodel_array_cropped==k)
if ( true_points_pred.size == 0 ):
invalid = True
HD_3D = float('inf')
print('invalid2')
else:
# only for rectum, only calculate the dice in the slices where there is a prediction (because it is not fully delineated in the real labels)
if "rect" in ROI:
top_left = true_points_pred.min(axis=0)
bottom_right = true_points_pred.max(axis=0)
label_3Dmodel_array_cropped = label_3Dmodel_array_cropped[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1]
reallabel_array_cropped = reallabel_array_cropped[top_left[0]:bottom_right[0]+1, top_left[1]:bottom_right[1]+1, top_left[2]:bottom_right[2]+1]
true_points_pred = np.argwhere(label_3Dmodel_array_cropped==k)
true_points_real = np.argwhere(reallabel_array_cropped==k)
if ( true_points_pred.size == 0 ) or (true_points_real.size==0):
invalid = True
HD_3D = float('inf')
print('invalid2')
else:
HD_3D = hd95(np.where(label_3Dmodel_array_cropped==k,1,0),np.where(reallabel_array_cropped==k,1,0),spacing,True)
else:
HD_3D = hd95(np.where(label_3Dmodel_array_cropped==k,1,0),np.where(reallabel_array_cropped==k,1,0),spacing,True)
# find HD ROI index
if "vessie" in ROI:
HD_ROI_ind = 0
elif "rect" in ROI:
HD_ROI_ind = 1
elif "prostate" in ROI:
HD_ROI_ind = 2
else:
HD_ROI_ind = -1
if (HD_ROI_ind != -1):
patient_HD_row[0,HD_ROI_ind] = HD_3D
#if (patient_id >= simu_testimages_range[0]) & (patient_id <= simu_testimages_range[1]):
# simu_HD_vector[patient_id - simu_testimages_range[0],HD_ROI_ind] = HD_3D
#elif (patient_id >= clb_testimages_range[0]) & (patient_id <= clb_testimages_range[1]):
# clb_HD_vector[patient_id - clb_testimages_range[0],HD_ROI_ind] = HD_3D
#elif (patient_id >= rennes_testimages_range[0]) & (patient_id <= rennes_testimages_range[1]):
# rennes_HD_vector[patient_id - rennes_testimages_range[0],HD_ROI_ind] = HD_3D
HD_3D_text=HD_3D_text+f'{HD_3D}\t'
#print(np.sum(label_3Dmodel_array[reallabel_array==k]==k)*2.0 )
#print(np.sum(label_3Dmodel_array[label_3Dmodel_array==k]==k) )
#print( np.sum(reallabel_array[reallabel_array==k]==k))
#print(HD_3D)
if not (invalid or math.isnan(HD_3D) or (HD_3D == float('inf'))):
meanHD_3D_perpatient = meanHD_3D_perpatient + HD_3D
num_labels_3D = num_labels_3D + 1
elif HD_ROI_ind != -1:
patient_count_invalids[HD_ROI_ind] = patient_count_invalids[HD_ROI_ind] + 1
if (patient_id >= simu_testimages_range[0]) & (patient_id <= simu_testimages_range[1]):
if np.size(simu_HD_vector) == 0:
simu_HD_vector = patient_HD_row
else:
simu_HD_vector = np.concatenate((simu_HD_vector,patient_HD_row),axis=0)
simu_count_invalids = simu_count_invalids + patient_count_invalids
#simu_HD_vector[patient_id - simu_testimages_range[0],HD_ROI_ind] = HD_3D
elif (patient_id >= clb_testimages_range[0]) & (patient_id <= clb_testimages_range[1]):
if np.size(clb_HD_vector) == 0:
clb_HD_vector = patient_HD_row
else:
clb_HD_vector = np.concatenate((clb_HD_vector,patient_HD_row),axis=0)
clb_count_invalids = clb_count_invalids + patient_count_invalids
#clb_HD_vector[patient_id - clb_testimages_range[0],HD_ROI_ind] = HD_3D
elif (patient_id >= rennes_testimages_range[0]) & (patient_id <= rennes_testimages_range[1]):
if np.size(rennes_HD_vector) == 0:
rennes_HD_vector = patient_HD_row
else:
rennes_HD_vector = np.concatenate((rennes_HD_vector,patient_HD_row),axis=0)
rennes_count_invalids = rennes_count_invalids + patient_count_invalids
#rennes_HD_vector[patient_id - rennes_testimages_range[0],HD_ROI_ind] = HD_3D
elif (patient_id >= pseudoCT_testimages_range[0]) & (patient_id <= pseudoCT_testimages_range[1]):
if np.size(pseudoCT_HD_vector) == 0:
pseudoCT_HD_vector = patient_HD_row
else:
pseudoCT_HD_vector = np.concatenate((pseudoCT_HD_vector,patient_HD_row),axis=0)
pseudoCT_count_invalids = pseudoCT_count_invalids + patient_count_invalids
#meanHD_2D_perpatient=meanHD_2D_perpatient/num_labels_2D
#meanHD_2D = meanHD_2D+meanHD_2D_perpatient
#HDscores_2D=HDscores_2D+f"{meanHD_2D_perpatient}\n"
#with open(HD_file_2D, "a") as f:
# f.write(HDscores_2D)
if num_labels_3D==0:
HD_3D_text=HD_3D_text+"nan\n"
else:
meanHD_3D_perpatient=meanHD_3D_perpatient/num_labels_3D
meanHD_3D = meanHD_3D+meanHD_3D_perpatient
HD_count = HD_count +1
HD_3D_text=HD_3D_text+f"{meanHD_3D_perpatient}\n"
with open(HD_file_3D, "a") as f:
f.write(HD_3D_text)
np.savetxt(f'{modelresults_folder}/simu_HD_vector.csv', simu_HD_vector, delimiter=',')
np.savetxt(f'{modelresults_folder}/clb_HD_vector.csv', clb_HD_vector, delimiter=',')
np.savetxt(f'{modelresults_folder}/rennes_HD_vector.csv', rennes_HD_vector, delimiter=',')
np.savetxt(f'{modelresults_folder}/pseudoCT_HD_vector.csv', pseudoCT_HD_vector, delimiter=',')
np.savetxt(f'{modelresults_folder}/simu_invalids_count_hd.csv', simu_count_invalids, delimiter=',')
np.savetxt(f'{modelresults_folder}/clb_invalids_count_hd.csv', clb_count_invalids, delimiter=',')
np.savetxt(f'{modelresults_folder}/rennes_invalids_count_hd.csv', rennes_count_invalids, delimiter=',')
np.savetxt(f'{modelresults_folder}/pseudoCT_invalids_count_hd.csv', pseudoCT_count_invalids, delimiter=',')
boxplot_array_vessie, boxplot_array_rect, boxplot_array_prostate, boxplot_labels, bladderboxplot_invalids_vector, rectumboxplot_invalids_vector, prostateboxplot_invalids_vector = build_boxplot_data(simu_HD_vector,clb_HD_vector,rennes_HD_vector,pseudoCT_HD_vector, simu_count_invalids, clb_count_invalids, rennes_count_invalids, pseudoCT_count_invalids)
plot_boxplot(boxplot_array_vessie,boxplot_labels, bladderboxplot_invalids_vector, f'Bladder HD scores (in mm) for test dataset', f'{modelresults_folder}/vessie_hausdorff.png')
plot_boxplot(boxplot_array_rect,boxplot_labels, rectumboxplot_invalids_vector, f'Rectum HD scores (in mm) for test dataset', f'{modelresults_folder}/rectum_hausdorff.png')
plot_boxplot(boxplot_array_prostate,boxplot_labels, prostateboxplot_invalids_vector, f'Prostate HD scores (in mm) for test dataset', f'{modelresults_folder}/prostate_hausdorff.png')
if 'CBCT' in modelresults_folder:
CBCT_simu_HD_vector = simu_HD_vector
CBCT_clb_HD_vector = clb_HD_vector
CBCT_rennes_HD_vector = rennes_HD_vector
# try to find the HD score vectors for the similar model trained with CBCT
CT_folder = modelresults_folder.replace('CBCT','CT')
if os.path.isfile(f'{CT_folder}/simu_HD_vector.csv'):
CT_simu_HD_vector = np.loadtxt(f'{CT_folder}/simu_HD_vector.csv', delimiter=',')
if os.path.isfile(f'{CT_folder}/clb_HD_vector.csv'):
CT_clb_HD_vector = np.loadtxt(f'{CT_folder}/clb_HD_vector.csv', delimiter=',')
if os.path.isfile(f'{CT_folder}/rennes_HD_vector.csv'):
CT_rennes_HD_vector = np.loadtxt(f'{CT_folder}/rennes_HD_vector.csv', delimiter=',')
CBCT_simu_invalids_vector = simu_count_invalids
CBCT_clb_invalids_vector = clb_count_invalids
CBCT_rennes_invalids_vector = rennes_count_invalids
# try to find the invalid vectors for the similar model trained with CBCT
CT_folder = modelresults_folder.replace('CBCT','CT')
if os.path.isfile(f'{CT_folder}/simu_invalids_count_hd.csv'):
CT_simu_invalids_vector = np.loadtxt(f'{CT_folder}/simu_invalids_count_hd.csv', delimiter=',')
if os.path.isfile(f'{CT_folder}/clb_invalids_count_hd.csv'):
CT_clb_invalids_vector = np.loadtxt(f'{CT_folder}/clb_invalids_count_hd.csv', delimiter=',')
if os.path.isfile(f'{CT_folder}/rennes_invalids_count_hd.csv'):
CT_rennes_invalids_vector = np.loadtxt(f'{CT_folder}/rennes_invalids_count_hd.csv', delimiter=',')
elif 'CT' in modelresults_folder:
# try to find the HD score vectors for the similar model trained with CBCT
#print(simu_HD_vector)
CT_simu_HD_vector = simu_HD_vector
CT_clb_HD_vector = clb_HD_vector
CT_rennes_HD_vector = rennes_HD_vector
CBCT_folder = modelresults_folder.replace('CT','CBCT')
if os.path.isfile(f'{CBCT_folder}/simu_HD_vector.csv'):
CBCT_simu_HD_vector = np.loadtxt(f'{CBCT_folder}/simu_HD_vector.csv', delimiter=',')
if os.path.isfile(f'{CBCT_folder}/clb_HD_vector.csv'):
CBCT_clb_HD_vector = np.loadtxt(f'{CBCT_folder}/clb_HD_vector.csv', delimiter=',')
if os.path.isfile(f'{CBCT_folder}/rennes_HD_vector.csv'):
CBCT_rennes_HD_vector = np.loadtxt(f'{CBCT_folder}/rennes_HD_vector.csv', delimiter=',')
CT_simu_invalids_vector = simu_count_invalids
CT_clb_invalids_vector = clb_count_invalids
CT_rennes_invalids_vector = rennes_count_invalids
# try to find the invalid vectors for the similar model trained with CBCT
CBCT_folder = modelresults_folder.replace('CT','CBCT')
if os.path.isfile(f'{CBCT_folder}/simu_invalids_count_hd.csv'):
CBCT_simu_invalids_vector = np.loadtxt(f'{CBCT_folder}/simu_invalids_count_hd.csv', delimiter=',')
if os.path.isfile(f'{CBCT_folder}/clb_invalids_count_hd.csv'):
CBCT_clb_invalids_vector = np.loadtxt(f'{CBCT_folder}/clb_invalids_count_hd.csv', delimiter=',')
if os.path.isfile(f'{CBCT_folder}/rennes_invalids_count_hd.csv'):
CBCT_rennes_invalids_vector = np.loadtxt(f'{CBCT_folder}/rennes_invalids_count_hd.csv', delimiter=',')
try:
# simu cbct
plot_boxplot([CBCT_simu_HD_vector[:,0], CT_simu_HD_vector[:,0]],['Trained with CBCT','Trained with CT'], [CBCT_simu_invalids_vector[0], CT_simu_invalids_vector[0]] , f'Bladder HD scores (in mm) for simulated cbct test dataset', f'{modelresults_folder}/simu_CTvsCBCT_vessie_hausdorff.png')
plot_boxplot([CBCT_simu_HD_vector[:,1], CT_simu_HD_vector[:,1]],['Trained with CBCT','Trained with CT'], [CBCT_simu_invalids_vector[1], CT_simu_invalids_vector[1]], f'Rectum HD scores (in mm) for simulated cbct test dataset', f'{modelresults_folder}/simu_CTvsCBCT_rectum_hausdorff.png')
plot_boxplot([CBCT_simu_HD_vector[:,2], CT_simu_HD_vector[:,2]],['Trained with CBCT','Trained with CT'], [CBCT_simu_invalids_vector[2], CT_simu_invalids_vector[2]], f'Prostate HD scores (in mm) for simulated cbct test dataset', f'{modelresults_folder}/simu_CTvsCBCT_prostate_hausdorff.png')
except:
print('Simu HD scores not available')
try:
# CLB cbct
plot_boxplot([CBCT_clb_HD_vector[:,0], CT_clb_HD_vector[:,0]],['Trained with CBCT','Trained with CT'],[CBCT_clb_invalids_vector[0], CT_clb_invalids_vector[0]], f'Bladder HD scores (in mm) for Lyon cbct test dataset', f'{modelresults_folder}/clb_CTvsCBCT_vessie_hausdorff.png')
plot_boxplot([CBCT_clb_HD_vector[:,1], CT_clb_HD_vector[:,1]],['Trained with CBCT','Trained with CT'],[CBCT_clb_invalids_vector[1], CT_clb_invalids_vector[1]], f'Rectum HD scores (in mm) for Lyon cbct test dataset', f'{modelresults_folder}/clb_CTvsCBCT_rectum_hausdorff.png')
plot_boxplot([CBCT_clb_HD_vector[:,2], CT_clb_HD_vector[:,2]],['Trained with CBCT','Trained with CT'],[CBCT_clb_invalids_vector[2], CT_clb_invalids_vector[2]], f'Prostate HD scores (in mm) for Lyon cbct test dataset', f'{modelresults_folder}/clb_CTvsCBCT_prostate_hausdorff.png')
except:
print('CLB HD scores not available')
try:
# Rennes cbct
plot_boxplot([CBCT_rennes_HD_vector[:,0], CT_rennes_HD_vector[:,0]],['Trained with CBCT','Trained with CT'], [CBCT_rennes_invalids_vector[0], CT_rennes_invalids_vector[0]], f'Bladder HD scores (in mm) for Rennes cbct test dataset', f'{modelresults_folder}/rennes_CTvsCBCT_vessie_hausdorff.png')
plot_boxplot([CBCT_rennes_HD_vector[:,1], CT_rennes_HD_vector[:,1]],['Trained with CBCT','Trained with CT'], [CBCT_rennes_invalids_vector[1], CT_rennes_invalids_vector[1]], f'Rectum HD scores (in mm) for Rennes cbct test dataset', f'{modelresults_folder}/rennes_CTvsCBCT_rectum_hausdorff.png')
plot_boxplot([CBCT_rennes_HD_vector[:,2], CT_rennes_HD_vector[:,2]],['Trained with CBCT','Trained with CT'], [CBCT_rennes_invalids_vector[2], CT_rennes_invalids_vector[2]], f'Prostate HD scores (in mm) for Rennes cbct test dataset', f'{modelresults_folder}/rennes_CTvsCBCT_prostate_hausdorff.png')
except:
print('Rennes HD scores not available')
# one box plot for all training and test datasets
try:
#plot_boxplot([CBCT_simu_HD_vector[:,0], CT_simu_HD_vector[:,0], CBCT_clb_HD_vector[:,0], CT_clb_HD_vector[:,0], CBCT_rennes_HD_vector[:,0], CT_rennes_HD_vector[:,0]],['simu CBCT\nsimu CBCT','real CT\nsimu CBCT','simu CBCT\nCLB CBCT','real CT\nCLB CBCT','simu CBCT\nCEM CBCT','real CT\nCEM CBCT'], f'Vessie HD scores for all cbct test subsets', f'{modelresults_folder}/CTvsCBCT_vessie_hausdorff.png')
plot_boxplot([CBCT_simu_HD_vector[:,0], CT_simu_HD_vector[:,0], CBCT_clb_HD_vector[:,0], CT_clb_HD_vector[:,0], CBCT_rennes_HD_vector[:,0], CT_rennes_HD_vector[:,0]],['simu CBCT','real CT','simu CBCT','real CT','simu CBCT','real CT'], [CBCT_simu_invalids_vector[0], CT_simu_invalids_vector[0],CBCT_clb_invalids_vector[0], CT_clb_invalids_vector[0],CBCT_rennes_invalids_vector[0], CT_rennes_invalids_vector[0]], f'Bladder HD scores (in mm) for all test CBCT subsets', f'{modelresults_folder}/CTvsCBCT_vessie_hausdorff.png')
plot_boxplot([CBCT_simu_HD_vector[:,1], CT_simu_HD_vector[:,1], CBCT_clb_HD_vector[:,1], CT_clb_HD_vector[:,1], CBCT_rennes_HD_vector[:,1], CT_rennes_HD_vector[:,1]],['simu CBCT','real CT','simu CBCT','real CT','simu CBCT','real CT'], [CBCT_simu_invalids_vector[1], CT_simu_invalids_vector[1],CBCT_clb_invalids_vector[1], CT_clb_invalids_vector[1],CBCT_rennes_invalids_vector[1], CT_rennes_invalids_vector[1]], f'Rectum HD scores (in mm) for all test CBCT subsets', f'{modelresults_folder}/CTvsCBCT_rectum_hausdorff.png')
plot_boxplot([CBCT_simu_HD_vector[:,2], CT_simu_HD_vector[:,2], CBCT_clb_HD_vector[:,2], CT_clb_HD_vector[:,2], CBCT_rennes_HD_vector[:,2], CT_rennes_HD_vector[:,2]],['simu CBCT','real CT','simu CBCT','real CT','simu CBCT','real CT'], [CBCT_simu_invalids_vector[2], CT_simu_invalids_vector[2],CBCT_clb_invalids_vector[2], CT_clb_invalids_vector[2],CBCT_rennes_invalids_vector[2], CT_rennes_invalids_vector[2]], f'Prostate HD scores (in mm) for all test CBCT subsets', f'{modelresults_folder}/CTvsCBCT_prostate_hausdorff.png')
except:
print('HD scores not available')
# calculate mean of means
#with open(HD_file_2D, "a") as f:
# f.write(f"Total mean: {meanHD_2D/num_patients}\n")
with open(HD_file_3D, "a") as f:
f.write(f"Total mean: {meanHD_3D/HD_count}\n")
import os
from medpy.metric.binary import hd95
import SimpleITK as sitk
import pkg_resources
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='Hausdorff95', formatter_class=argparse.RawTextHelpFormatter,
description='\nThis code is used to get the Hausdorff 95th percentile. '+\
'For questions and feedback contact: [email protected]')
parser.add_argument('-gt', dest='groundTruth', type=str,
help='The ground truth image for comparison.\n',
required=True)
parser.add_argument('-m', dest='maskImage', type=str,
help='The annotated mask to compare against ground truth.\n',
required=True)
# parser.add_argument('-v', '--version', action='version',
# version=pkg_resources.require("Hausdorff95")[0].version, help="Show program's version number and exit.") # disabled because pyinstaller doesn't import it properly
args = parser.parse_args()
groundTruth = os.path.abspath(args.groundTruth)
maskImage = os.path.abspath(args.maskImage)
gt = sitk.GetArrayFromImage(sitk.ReadImage(groundTruth))
mk = sitk.GetArrayFromImage(sitk.ReadImage(maskImage))
print(hd95(gt,mk))
# list of output segmentations
list_of_output = ['', '', '', ......]
all_assd = []
all_hd95 = []
for x in range(len(list_of_gt)):
gt_label = '/path/to/current/groundtruth/segmentation.nii.gz'
gt_path = read_nifti(gt_label)
read_segm_output = read_nifti(segm_output)
segm_output_cc = getLargestCC(read_segm_output)
segm_output_cc_path = '/path/to/current/segmentation/output.nii.gz'
save_nifti(segm_output_cc, segm_output_cc_path)
result, h1 = load(segm_output_cc_path)
reference, h2 = load(gt_path)
all_hd95.append(binary.hd95(reference, result, voxelspacing =voxelspacing))
all_assd.append(binary.assd(reference, result, voxelspacing =voxelspacing))
print(">>>>>>>")
print("hd95 for all subjects", np.mean(all_hd95), np.std(all_hd95))
print(">>>>>>>")
print("assd for all subjects", np.mean(all_assd), np.std(all_assd))
print(">>>>>>>")
print("asd for all subjects", np.mean(all_asd), np.std(all_asd))
