def do(argv):
segResults_path = argv[0]
references_path = argv[1]
num_classes = argv[2]
results = listdir(segResults_path)
results.sort(key=lambda f: int(filter(str.isdigit, f)))
references = listdir(references_path)
references.sort(key=lambda f: int(filter(str.isdigit, f)))
pack = zip(results, references)
res = 1
for resultname, referencename in pack:
result = nib.load(join(segResults_path, resultname)).get_data()
print('result name: ', resultname)
print('reference name: ', referencename)
reference = nib.load(join(references_path, referencename)).get_data()
reference = reference.reshape(result.shape)
print("ASSD: ", mmb.assd(result, reference, voxelspacing=res))
Assd_array = []
Dice_array = []
print(num_classes)
for res_c, ref_c in zip(
range(num_classes), range(num_classes)
): #or some specific class labels, e.g., zip([2,3,4], [2,3,4])
dc = mmb.dc(result == res_c, reference == ref_c)
assd = mmb.assd(result == res_c,
reference == ref_c,
voxelspacing=res)
Assd_array.append(assd)
Dice_array.append(dc)
for i in xrange(0, len(Dice_array)):
print('Dice score of class_{}'.format(i + 1), Dice_array[i])
for i in xrange(0, len(Assd_array)):
print('ASSD score of class_{}'.format(i + 1), Assd_array[i])
#print("obj_assd: ",obj_assd(result,reference))
#print("ASD: ", asd(result,reference))
#print("ASD: ", asd(reference,result))
#print("HD: ", hd(result,reference))
print("Dice: ", mmb.dc(result, reference))
def get_outputs(seg, seg_truth):
print seg.shape, seg_truth.shape
seg_thresh = utility.threshold(seg,THRESHOLD)
print seg_thresh.shape
contour = utility.listSegToContours(seg_thresh, meta_test[1,:],
meta_test[0,:], ISOVALUE)
errs = utility.listAreaOverlapError(contour, contours_test)
thresh,ts = utility.cum_error_dist(errs,DX)
roc = roc_curve(np.ravel(seg_truth),np.ravel(seg), pos_label=1)
pr = precision_recall_curve(np.ravel(seg_truth),np.ravel(seg), pos_label=1)
dorf = []
emd = []
asdl = []
dice = []
prec = []
for i in range(len(seg_truth)):
if np.sum(seg_thresh[i,:,:]) > 0.1 and np.sum(seg_truth[i,:,:,0]) > 0.1:
e= hd(seg_thresh[i,:,:],seg_truth[i,:,:,0],meta_test[0,i][0])
dorf.append(e)
e_asd= assd(seg_thresh[i,:,:],seg_truth[i,:,:,0],meta_test[0,i][0])
asdl.append(e_asd)
# if np.sum(seg_thresh[i,:,:]) < 600 and np.sum(seg_truth[i,:,:,0]) < 600:
# print i,np.sum(seg_thresh[i,:,:]),np.sum(seg_truth[i,:,:,0])
# e_emd = utility.EMDSeg(seg_truth[i,:,:,0],seg_thresh[i,:,:], meta_test[0,i][0])
# emd.append(e_emd)
edc = dc(seg_thresh[i,:,:],seg_truth[i,:,:,0])
dice.append(edc)
prec.append(precision(seg_thresh[i,:,:],seg_truth[i,:,:,0]))
acc,mean_acc = calc_accuracy(seg, seg_truth)
return (contour,errs,thresh,roc,pr,acc,mean_acc,dorf,dice, prec,asdl)
def performance(output, target):
pos_probs = torch.sigmoid(output)
pos_preds = (pos_probs > 0.5).float()
pos_preds = pos_preds.cpu().numpy().squeeze()
target = target.cpu().numpy().squeeze()
if target.sum() == 0: # background patch
return 0, 0
try:
# ACD
acd_se = binary.assd(pos_preds, target)
# ASD
d_sg = np.sqrt(binary.__surface_distances(pos_preds, target, 1))
d_gs = np.sqrt(binary.__surface_distances(target, pos_preds, 1))
asd_se = (d_sg.sum() + d_gs.sum()) / (len(d_sg) + len(d_gs))
except:
#pred == 0
acd_se = None
asd_se = None
# IoU
union = ((pos_preds + target) != 0).sum()
intersection = (pos_preds * target).sum()
iou = intersection / union
# dice
dice = (2 * intersection) / (pos_preds.sum() + target.sum())
return iou, dice, acd_se, asd_se
def update_eval_metrics(preds, labels, eval_metrics):
if len(labels.shape) == 2:
preds = np.expand_dims(preds, axis=0)
labels = np.expand_dims(labels, axis=0)
N = labels.shape[0]
for i in range(N):
pred = preds[i, :, :]
label = labels[i, :, :]
eval_metrics['dice score'].append(dc(pred, label))
eval_metrics['precision'].append(precision(pred, label))
eval_metrics['recall'].append(recall(pred, label))
eval_metrics['sensitivity'].append(sensitivity(pred, label))
eval_metrics['specificity'].append(specificity(pred, label))
if np.sum(pred) > 0 and np.sum(label) > 0:
eval_metrics['hausdorff'].append(hd(pred, label))
eval_metrics['hausdorff 95%'].append(hd95(pred, label))
eval_metrics['asd'].append(asd(pred, label))
eval_metrics['assd'].append(assd(pred, label))
eval_metrics['jaccard'].append(jc(pred, label))
else:
eval_metrics['hausdorff'].append('nan')
eval_metrics['hausdorff 95%'].append('nan')
eval_metrics['asd'].append('nan')
eval_metrics['assd'].append('nan')
eval_metrics['jaccard'].append('nan')
return eval_metrics
def performance_by_slice(output_list, target_list,img_name_list):
assert len(output_list) == len(target_list), 'not same list lenths'
performance = {}
for i in range(len(output_list)):
preds = output_list[i]
slice_pred = (preds > 0.5).astype('float')
slice_target = target_list[i]
# slice-level classification performance
tp = fp = tn = fn = 0
is_gt_positive = slice_target.max()
is_pred_positive = slice_pred.max()
if is_gt_positive:
if is_pred_positive:
tp = 1
else:
fn = 1
else:
if is_pred_positive:
fp = 1
else:
tn = 1
# slice-level segmentation performance
iou = dice = -1
if is_gt_positive:
union = ((slice_pred + slice_target) != 0).sum()
intersection = (slice_pred * slice_target).sum()
iou = intersection / union
dice = (2 * intersection) / (slice_pred.sum() + slice_target.sum())
try:
# ACD
acd_se = binary.assd(slice_pred, slice_target)
# ASD
d_sg = np.sqrt(binary.__surface_distances(slice_pred, slice_target, 1))
d_gs = np.sqrt(binary.__surface_distances(slice_target, slice_pred, 1))
asd_se = (d_sg.sum() + d_gs.sum()) / (len(d_sg) + len(d_gs))
except:
# pred == 0
acd_se = None
asd_se = None
# TODO: not need to store gt and pred
performance[str(i)] = {'cls': [tp, fp, tn, fn],
'seg': [iou, dice],
'gt': slice_target,
'pred': slice_pred,
'img':img_name_list[i],
'acd_se':acd_se,
'asd_se': asd_se,
}
#'pixel': [gt_pixel, pred_pixel],
return performance
def compute_metrics_on_directories_raw(dir_gt, dir_pred):
"""
Calculates all possible metrics (the ones from the metrics script as well as
hausdorff and average symmetric surface distances)
:param dir_gt: Directory of the ground truth segmentation maps.
:param dir_pred: Directory of the predicted segmentation maps.
:return:
"""
lst_gt = sorted(glob(os.path.join(dir_gt, '*')), key=natural_order)
lst_pred = sorted(glob(os.path.join(dir_pred, '*')), key=natural_order)
res = []
cardiac_phase = []
file_names = []
measure_names = [
'Dice LV', 'Volume LV', 'Err LV(ml)', 'Dice RV', 'Volume RV',
'Err RV(ml)', 'Dice MYO', 'Volume MYO', 'Err MYO(ml)', 'Hausdorff LV',
'Hausdorff RV', 'Hausdorff Myo', 'ASSD LV', 'ASSD RV', 'ASSD Myo'
]
res_mat = np.zeros((len(lst_gt), len(measure_names)))
ind = 0
for p_gt, p_pred in zip(lst_gt, lst_pred):
if os.path.basename(p_gt) != os.path.basename(p_pred):
raise ValueError("The two files don't have the same name"
" {}, {}.".format(os.path.basename(p_gt),
os.path.basename(p_pred)))
gt, _, header = load_nii(p_gt)
pred, _, _ = load_nii(p_pred)
zooms = header.get_zooms()
res.append(metrics(gt, pred, zooms))
cardiac_phase.append(
os.path.basename(p_gt).split('.nii.gz')[0].split('_')[-1])
file_names.append(os.path.basename(p_pred))
res_mat[ind, :9] = metrics(gt, pred, zooms)
for ii, struc in enumerate([3, 1, 2]):
gt_binary = (gt == struc) * 1
pred_binary = (pred == struc) * 1
res_mat[ind, 9 + ii] = hd(gt_binary,
pred_binary,
voxelspacing=zooms,
connectivity=1)
res_mat[ind, 12 + ii] = assd(pred_binary,
gt_binary,
voxelspacing=zooms,
connectivity=1)
ind += 1
return res_mat, cardiac_phase, measure_names, file_names
def do(argv):
segResults_path = argv[0]
references_path = argv[1]
num_classes = argv[2]
results = listdir(segResults_path)
results.sort(key=lambda f: int(filter(str.isdigit, f)))
references = listdir(references_path)
references.sort(key=lambda f: int(filter(str.isdigit, f)))
res = 1 # image-specific variable
for resultname, referencename in zip(results,references):
result = nib.load(join(segResults_path,resultname)).get_data()
reference = nib.load(join(references_path,referencename)).get_data()
# Please adjust this for your own data
reference = numpy.expand_dims(reference,axis=3)
reference = numpy.expand_dims(reference,axis=4)
print("ASSD: ", mmb.assd(result,reference,voxelspacing=res))
Assd_array = []
Dice_array = []
print(num_classes)
for c_i in xrange(0,int(num_classes)):
dc = mmb.dc(result == c_i, reference == c_i)
assd = mmb.assd(result == c_i, reference == c_i,voxelspacing=res)
Assd_array.append(assd)
Dice_array.append(dc)
for i in xrange(0, len(Dice_array)):
print('Dice score of class_{}'.format(i), Dice_array[i])
for i in xrange(0,len(Assd_array)):
print('ASSD score of class_{}'.format(i), Assd_array[i])
#print("obj_assd: ",obj_assd(result,reference))
#print("ASD: ", asd(result,reference))
#print("ASD: ", asd(reference,result))
#print("HD: ", hd(result,reference))
print("Dice: ", mmb.dc(result,reference))
def compute_assd(pred_label, gt_label):
assert (pred_label.shape == gt_label.shape)
assd_info = []
for ii in range(pred_label.shape[0]):
if (pred_label[ii].sum() == 0 or gt_label[ii].sum() == 0):
continue
assd_dist = assd(pred_label[ii], gt_label[ii])
assd_info.append(assd_dist)
return assd_info
def binary_measures_numpy(result, target, binary_threshold=0.5):
result_binary = (result > binary_threshold).astype(numpy.uint8)
target_binary = (target > binary_threshold).astype(numpy.uint8)
result = BinaryMeasuresDto(mpm.dc(result_binary, target_binary), numpy.Inf,
numpy.Inf,
mpm.precision(result_binary, target_binary),
mpm.sensitivity(result_binary, target_binary),
mpm.specificity(result_binary, target_binary))
if result_binary.any() and target_binary.any():
result.hd = mpm.hd(result_binary, target_binary)
result.assd = mpm.assd(result_binary, target_binary)
return result
def process_model(c,files,mhas):
errs = []
dorf = []
asd = []
ravd_arr = []
dc_arr = []
vols = []
for f in files:
mod = f.replace('truth',c)
img_name = f.split('.')[0]
img_file = [i for i in mhas if img_name in i][0]
print f, mod, img_file
ref_img = sitk.ReadImage(img_file)
p1 = utility.readVTKPD(vtk_dir+f)
p2 = utility.readVTKPD(vtk_dir+mod)
e = utility.jaccard3D_pd_to_itk(p1,p2,ref_img)
errs.append(e)
np1 = utility.pd_to_numpy_vol(p1, spacing=ref_img.GetSpacing(), shape=ref_img.GetSize(), origin=ref_img.GetOrigin() )
np2 = utility.pd_to_numpy_vol(p2, spacing=ref_img.GetSpacing(), shape=ref_img.GetSize(), origin=ref_img.GetOrigin() )
if np.sum(np1) > 0 and np.sum(np2) > 0:
e = hd(np1,np2, ref_img.GetSpacing()[0])
dorf.append(e)
e_asd = assd(np1,np2, ref_img.GetSpacing()[0])
asd.append(e_asd)
e_ravd = abs(ravd(np1,np2))
ravd_arr.append(e_asd)
e_dc = dc(np1,np2)
dc_arr.append(e_dc)
vols.append(np.sum(np1)*ref_img.GetSpacing()[0])
np.save(output_dir+'{}.jaccard.npy'.format(c),errs)
np.save(output_dir+'{}.dorf.npy'.format(c),dorf)
np.save(output_dir+'{}.assd.npy'.format(c),asd)
np.save(output_dir+'{}.ravd.npy'.format(c),ravd_arr)
np.save(output_dir+'{}.dc.npy'.format(c),dc_arr)
np.save(output_dir+'{}.vols.npy'.format(c),vols)
return '{} , {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\n'.format(\
c,np.mean(errs),np.std(errs),np.mean(dorf),np.std(dorf),np.mean(asd),\
np.std(asd), np.mean(ravd_arr),np.std(ravd_arr),np.mean(dc_arr),np.std(dc_arr))
def assds_each_class(prediction,
target,
class_num=20,
eps=1e-10,
voxel_size=(1, 1, 1),
empty_value=-1.0,
connectivity=1):
assds = empty_value * np.ones((class_num), dtype=np.float32)
for i in range(0, class_num):
if i not in target:
continue
if i not in prediction:
print('label %d is zero' % i)
continue
target_per_class = np.where(target == i, 1, 0)
prediction_per_class = np.where(prediction == i, 1, 0)
ad = assd(prediction_per_class,
target_per_class,
voxelspacing=voxel_size,
connectivity=connectivity)
assds[i] = ad
return assds
def get_assd_metric(batch_result, batch_reference, pixel_size_mm=1,
by_label=True):
"""Computes the Average Symmetric Surface Distance (ASSD) between
two masks from two batches."""
if batch_result.shape != batch_reference.shape:
raise ValueError('The input batches must be the same size')
batch_result = get_one_hot_encoding_from_hard_segm(
np.asarray(batch_result),
labels=np.unique(batch_reference)).astype(np.bool)
batch_reference = get_one_hot_encoding_from_hard_segm(
np.asarray(batch_reference)).astype(np.bool)
dims = batch_result.shape
assd_values = np.ndarray(shape=(dims[0], dims[-1] - 1), dtype=np.float32)
for i in range(dims[0]):
for l in range(1, dims[-1]):
assd_values[i, l - 1] = assd(
batch_result[i, :, :, l], batch_reference[i, :, :, l])
if not by_label:
assd_values = np.mean(assd_values, axis=1)
return assd_values * float(pixel_size_mm)
def compute_scores(preds, labels):
preds_data = preds.data.cpu().numpy()
labels_data = labels.data.cpu().numpy()
dice_score = dc(preds_data, labels_data)
jaccard_coef = jc(preds_data, labels_data)
hausdorff_dist = hd(preds_data, labels_data)
asd_score = asd(preds_data, labels_data)
assd_score = assd(preds_data, labels_data)
precision_value = precision(preds_data, labels_data)
recall_value = recall(preds_data, labels_data)
sensitivity_value = sensitivity(preds_data, labels_data)
specificity_value = specificity(preds_data, labels_data)
return {
'dice score': dice_score,
'jaccard': jaccard_coef,
'hausdorff': hausdorff_dist,
'asd': asd_score,
'assd': assd_score,
'precision': precision_value,
'recall': recall_value,
'sensitivity': sensitivity_value,
'specificity': specificity_value
}
def get_geometrical_results(model, data, groundtruths, path, inds, use_pp):
"""Compute Dices, Hausdorff distances and ASSDs between the model predictions on the inds indices of data and the groundtruths
use_pp enables to use post-processing on the predictions. The original image sizes are stored in inds"""
# Get predictions from the input data
segmentations = model.predict(data, verbose=1)
predictions = np.argmax(segmentations, axis=-1)
voxel_spacing = [0.154, 0.308]
nb_im = predictions.shape[0]
nb_struct = groundtruths.shape[-1] - 1
results = np.zeros((nb_im, nb_struct, 3), dtype=np.float)
size_file = open(join(path, 'ori_sizes.txt'), 'r')
sizes = size_file.readlines()
# get image indices
indices = []
for ind in inds:
indices.append(ind * 2)
indices.append(ind * 2 + 1)
for im in tnrange(nb_im, desc="Evaluation on each image"):
# Reshape the prediction as a binary class matrix
pred_cat = keras.utils.to_categorical(predictions[im, :, :],
num_classes=4)
# get original size from the relative indexing
line = sizes[indices[im] - 600]
ori_size = [int(line.split()[2][1:-1]), int(line.split()[3][0:-1])]
for struct in range(1, nb_struct + 1):
# Extract the structure channel
pred = pred_cat[:, :, struct]
gt = groundtruths[im, :, :, struct]
# Resize to the original size
pred = np.array(
Image.fromarray(pred).resize(
(ori_size[1], ori_size[0]), PIL.Image.NEAREST)).astype(
'uint8') #Image.resize inverts dimensions
gt = np.array(
Image.fromarray(gt).resize((ori_size[1], ori_size[0]),
PIL.Image.NEAREST)).astype('uint8')
# Transform into boolean array
pred = (pred == 1)
gt = (gt == 1)
# Apply postprocessing
if use_pp:
pred = post_process(pred)
# Compute geometrical metrics using the medpy library
dice = dc(pred, gt)
if np.sum(
pred
) > 0: # If the structure is predicted on at least one pixel
hausdorff = hd(
pred,
gt,
voxelspacing=[voxel_spacing[0], voxel_spacing[1]])
asd = assd(pred,
gt,
voxelspacing=[voxel_spacing[0], voxel_spacing[1]])
else:
print("on image ", nb_im, "the structure ", struct,
" was not associated to any pixel")
hausdorff = 'nan'
asd = 'nan'
results[im, struct - 1, :] = [dice, hausdorff, asd]
return results, segmentations
def val_step(val_loader,
model,
criterion,
weights_criterion,
multiclass_criterion,
binary_threshold,
num_classes=4,
generate_stats=False,
generate_overlays=False,
save_path="",
swap_values=None):
"""
Perform a validation step
:param val_loader: (Dataloader) Validation dataset loader
:param model: Model used in training
:param criterion: Choosed criterion
:param weights_criterion: (list -> float) Choosed criterion weights
:param multiclass_criterion:
:param binary_threshold: (float) Threshold to set class as class 1. Typically 0.5
:param num_classes: Num total classes in predictions
:param generate_stats: (bool) If true generate predictions stats via pandas
:param generate_overlays: (bool) If true save mask predictions
:param save_path: (string) If save_preds then which directory to save mask predictions
:param swap_values: (list of lists) In predicted mask, swaps first item and second in list. Example: [[1,2]]
:return: Intersection Over Union and Dice Metrics, Mean validation loss
"""
# https://stackoverflow.com/questions/8713620/appending-items-to-a-list-of-lists-in-python
ious, dices = [[] for _ in range(num_classes)
], [[] for _ in range(num_classes)]
hausdorffs, assds = [[] for _ in range(num_classes)
], [[] for _ in range(num_classes)]
val_loss, df_info = [], []
model.eval()
if save_path != "":
os.makedirs(save_path, exist_ok=True)
with torch.no_grad():
for sample_indx, (image, original_img, original_mask, mask,
img_id) in enumerate(val_loader):
original_mask = original_mask.cuda()
image = image.type(torch.float).cuda()
prob_pred = model(image)
# ToDo: Fix val loss
# val_loss.append(calculate_loss(mask, prob_pred, criterion, weights_criterion, multiclass_criterion).item())
val_loss.append(0.0)
for indx, single_pred in enumerate(prob_pred):
original_mask = original_mask[indx].data.cpu().numpy().astype(
np.uint8).squeeze()
# Calculate metrics resizing prediction to original mask shape
pred_mask = convert_multiclass_mask(
single_pred.unsqueeze(0)).data.cpu().numpy()
pred_mask = reshape_masks(pred_mask.squeeze(0),
original_mask.shape)
if swap_values is not None:
for swap_val in swap_values:
pred_mask = np.where(
pred_mask == swap_val[0], swap_val[1],
np.where(pred_mask == swap_val[1], swap_val[0],
pred_mask))
patient_iou = [-1, -1, -1] # LV, MYO, RV
patient_dice = [-1, -1, -1] # LV, MYO, RV
patient_hausddorf = [999, 999, 999] # LV, MYO, RV
patient_assd = [999, 999, 999] # LV, MYO, RV
for current_class in np.unique(
np.concatenate((original_mask, pred_mask))):
binary_ground_truth = np.where(
original_mask == current_class, 1, 0).astype(np.int32)
binary_pred_mask = np.where(pred_mask == current_class, 1,
0).astype(np.int32)
tmp_iou = jaccard_coef(binary_ground_truth,
binary_pred_mask)
tmp_dice = dice_coef(binary_ground_truth, binary_pred_mask)
# https://github.com/loli/medpy/blob/master/medpy/metric/binary.py#L1212
if not np.count_nonzero(
binary_pred_mask) and not np.count_nonzero(
binary_ground_truth):
# The same, distances 0
tmp_assd = 0
tmp_hausdorff = 0
elif not np.count_nonzero(
binary_pred_mask) or not np.count_nonzero(
binary_ground_truth):
# ToDo: equivalent worst value for Hausdorff and surface distances
tmp_assd = 999
tmp_hausdorff = 999
else:
tmp_assd = medmetrics.assd(binary_pred_mask,
binary_ground_truth)
tmp_hausdorff = medmetrics.hd(binary_pred_mask,
binary_ground_truth)
ious[current_class].append(tmp_iou)
dices[current_class].append(tmp_dice)
hausdorffs[current_class].append(tmp_hausdorff)
assds[current_class].append(tmp_assd)
# -1 Due index 0 is background
if current_class != 0:
patient_iou[current_class - 1] = tmp_iou
patient_dice[current_class - 1] = tmp_dice
patient_hausddorf[current_class - 1] = tmp_hausdorff
patient_assd[current_class - 1] = tmp_assd
if generate_stats:
patient_info = img_id[0].split("_")
df_info.append({
"patient":
patient_info[0],
"slice":
patient_info[1][5:],
"phase":
patient_info[2][5:],
"IOU_LV":
patient_iou[0],
"IOU_MYO":
patient_iou[1],
"IOU_RV":
patient_iou[2],
"IOU_MEAN":
np.mean(
np.array([
value for value in patient_iou if value != -1
])),
"DICE_LV":
patient_dice[0],
"DICE_MYO":
patient_dice[1],
"DICE_RV":
patient_dice[2],
"DICE_MEAN":
np.mean(
np.array([
value for value in patient_dice if value != -1
])),
"HAUSDORFF_LV":
patient_hausddorf[0],
"HAUSDORFF_MYO":
patient_hausddorf[1],
"HAUSDORFF_RV":
patient_hausddorf[2],
"HAUSDORFF_MEAN":
np.mean(
np.array([
value for value in patient_hausddorf
if value != -999
])),
"ASSD_LV":
patient_assd[0],
"ASSD_MYO":
patient_assd[1],
"ASSD_RV":
patient_assd[2],
"ASSD_MEAN":
np.mean(
np.array([
value for value in patient_assd
if value != -999
])),
})
if generate_overlays:
plot_save_pred(original_img.data.cpu().numpy().squeeze(),
original_mask, pred_mask, save_path,
img_id[0])
stats = None
if generate_stats:
stats = pd.DataFrame(df_info)
stats.to_csv(os.path.join(save_path, "val_patient_stats.csv"),
index=False)
iou = [np.mean(i) for i in ious] # Class mean, not global
iou = np.append(
np.mean(iou[1:]),
iou) # Add as first value global class mean (without background)
dice = [np.mean(i) for i in dices]
dice = np.append(np.mean(dice[1:]), dice)
hausdorff = [np.mean(i) for i in hausdorffs]
hausdorff = np.append(np.mean(hausdorff[1:]), hausdorff)
assd = [np.mean(i) for i in assds]
assd = np.append(np.mean(assd[1:]), assd)
return iou, dice, hausdorff, assd, np.mean(val_loss), stats
def get_hd_assd(self, b):
hd95_dist = hd95(self.pred_mid_rl[b], self.gt_mid[b])
assd_dist = assd(self.pred_mid_rl[b], self.gt_mid[b])
return hd95_dist * self.ps, assd_dist * self.ps
# list of output segmentations
list_of_output = ['', '', '', ......]
all_assd = []
all_hd95 = []
for x in range(len(list_of_gt)):
gt_label = '/path/to/current/groundtruth/segmentation.nii.gz'
gt_path = read_nifti(gt_label)
read_segm_output = read_nifti(segm_output)
segm_output_cc = getLargestCC(read_segm_output)
segm_output_cc_path = '/path/to/current/segmentation/output.nii.gz'
save_nifti(segm_output_cc, segm_output_cc_path)
result, h1 = load(segm_output_cc_path)
reference, h2 = load(gt_path)
all_hd95.append(binary.hd95(reference, result, voxelspacing =voxelspacing))
all_assd.append(binary.assd(reference, result, voxelspacing =voxelspacing))
print(">>>>>>>")
print("hd95 for all subjects", np.mean(all_hd95), np.std(all_hd95))
print(">>>>>>>")
print("assd for all subjects", np.mean(all_assd), np.std(all_assd))
print(">>>>>>>")
print("asd for all subjects", np.mean(all_asd), np.std(all_asd))
def test(self):
"""Test Function."""
self.model_setup()
saver = tf.train.Saver()
init = tf.global_variables_initializer()
test_list = self.read_lists(self.test_fid)
with tf.Session() as sess:
sess.run(init)
saver.restore(sess, self.checkpoint_pth)
dice_list = []
assd_list = []
for idx_file, fid in enumerate(test_list):
_npz_dict = np.load(fid)
data = _npz_dict['arr_0']
label = _npz_dict['arr_1']
# This is to make the orientation of test data match with the training data
# Set to False if the orientation of test data has already been aligned with the training data
if True:
data = np.flip(data, axis=0)
data = np.flip(data, axis=1)
label = np.flip(label, axis=0)
label = np.flip(label, axis=1)
tmp_pred = np.zeros(label.shape)
frame_list = [kk for kk in range(data.shape[2])]
for ii in range(int(np.floor(data.shape[2] // self.batch_size))):
data_batch = np.zeros([self.batch_size, data_size[0], data_size[1], data_size[2]])
label_batch = np.zeros([self.batch_size, label_size[0], label_size[1]])
for idx, jj in enumerate(frame_list[ii * self.batch_size: (ii + 1) * self.batch_size]):
data_batch[idx, ...] = np.expand_dims(data[..., jj].copy(), 3)
label_batch[idx, ...] = label[..., jj].copy()
label_batch = self.label_decomp(label_batch)
if TEST_MODALITY=='CT':
if USE_newstat:
data_batch = np.subtract(np.multiply(np.divide(np.subtract(data_batch, -2.8), np.subtract(3.2, -2.8)), 2.0),1) # {-2.8, 3.2} need to be changed according to the data statistics
else:
data_batch = np.subtract(np.multiply(np.divide(np.subtract(data_batch, -1.9), np.subtract(3.0, -1.9)), 2.0),1) # {-1.9, 3.0} need to be changed according to the data statistics
elif TEST_MODALITY=='MR':
data_batch = np.subtract(np.multiply(np.divide(np.subtract(data_batch, -1.8), np.subtract(4.4, -1.8)), 2.0),1) # {-1.8, 4.4} need to be changed according to the data statistics
compact_pred_b_val = sess.run(self.compact_pred_b, feed_dict={self.input_b: data_batch, self.gt_b: label_batch})
for idx, jj in enumerate(frame_list[ii * self.batch_size: (ii + 1) * self.batch_size]):
tmp_pred[..., jj] = compact_pred_b_val[idx, ...].copy()
for c in range(1, self._num_cls):
pred_test_data_tr = tmp_pred.copy()
pred_test_data_tr[pred_test_data_tr != c] = 0
pred_gt_data_tr = label.copy()
pred_gt_data_tr[pred_gt_data_tr != c] = 0
dice_list.append(mmb.dc(pred_test_data_tr, pred_gt_data_tr))
assd_list.append(mmb.assd(pred_test_data_tr, pred_gt_data_tr))
dice_arr = 100 * np.reshape(dice_list, [4, -1]).transpose()
dice_mean = np.mean(dice_arr, axis=1)
dice_std = np.std(dice_arr, axis=1)
print 'Dice:'
print 'AA :%.1f(%.1f)' % (dice_mean[3], dice_std[3])
print 'LAC:%.1f(%.1f)' % (dice_mean[1], dice_std[1])
print 'LVC:%.1f(%.1f)' % (dice_mean[2], dice_std[2])
print 'Myo:%.1f(%.1f)' % (dice_mean[0], dice_std[0])
print 'Mean:%.1f' % np.mean(dice_mean)
assd_arr = np.reshape(assd_list, [4, -1]).transpose()
assd_mean = np.mean(assd_arr, axis=1)
assd_std = np.std(assd_arr, axis=1)
print 'ASSD:'
print 'AA :%.1f(%.1f)' % (assd_mean[3], assd_std[3])
print 'LAC:%.1f(%.1f)' % (assd_mean[1], assd_std[1])
print 'LVC:%.1f(%.1f)' % (assd_mean[2], assd_std[2])
print 'Myo:%.1f(%.1f)' % (assd_mean[0], assd_std[0])
print 'Mean:%.1f' % np.mean(assd_mean)
def compute_metrics_on_directories_raw(dir_gt, dir_pred):
"""
Calculates a number of measures from the predicted and ground truth segmentations:
- Dice
- Hausdorff distance
- Average surface distance
- Predicted volume
- Volume error w.r.t. ground truth
:param dir_gt: Directory of the ground truth segmentation maps.
:param dir_pred: Directory of the predicted segmentation maps.
:return: Pandas dataframe with all measures in a row for each prediction and each structure
"""
filenames_gt = sorted(glob(os.path.join(dir_gt, '*')), key=natural_order)
filenames_pred = sorted(glob(os.path.join(dir_pred, '*')), key=natural_order)
cardiac_phase = []
file_names = []
structure_names = []
# 5 measures per structure:
dices_list = []
hausdorff_list = []
assd_list = []
vol_list = []
vol_err_list = []
structures_dict = {1: 'RV', 2: 'Myo', 3: 'LV'}
for p_gt, p_pred in zip(filenames_gt, filenames_pred):
if os.path.basename(p_gt) != os.path.basename(p_pred):
raise ValueError("The two files don't have the same name"
" {}, {}.".format(os.path.basename(p_gt),
os.path.basename(p_pred)))
# load ground truth and prediction
gt, _, header = utils.load_nii(p_gt)
pred, _, _ = utils.load_nii(p_pred)
zooms = header.get_zooms()
# calculate measures for each structure
for struc in [3,1,2]:
gt_binary = (gt == struc) * 1
pred_binary = (pred == struc) * 1
volpred = pred_binary.sum() * np.prod(zooms) / 1000.
volgt = gt_binary.sum() * np.prod(zooms) / 1000.
vol_list.append(volpred)
vol_err_list.append(volpred - volgt)
if np.sum(gt_binary) == 0 and np.sum(pred_binary) == 0:
dices_list.append(1)
assd_list.append(0)
hausdorff_list.append(0)
elif np.sum(pred_binary) > 0 and np.sum(gt_binary) == 0 or np.sum(pred_binary) == 0 and np.sum(gt_binary) > 0:
logging.warning('Structure missing in either GT (x)or prediction. ASSD and HD will not be accurate.')
dices_list.append(0)
assd_list.append(1)
hausdorff_list.append(1)
else:
hausdorff_list.append(hd(gt_binary, pred_binary, voxelspacing=zooms, connectivity=1))
assd_list.append(assd(pred_binary, gt_binary, voxelspacing=zooms, connectivity=1))
dices_list.append(dc(gt_binary, pred_binary))
cardiac_phase.append(os.path.basename(p_gt).split('.nii.gz')[0].split('_')[-1])
file_names.append(os.path.basename(p_pred))
structure_names.append(structures_dict[struc])
df = pd.DataFrame({'dice': dices_list, 'hd': hausdorff_list, 'assd': assd_list,
'vol': vol_list, 'vol_err': vol_err_list,
'phase': cardiac_phase, 'struc': structure_names, 'filename': file_names})
return df
