def evaluate(model, device, dataset, args, training=False):
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.makedirs(args.output_dir)
eval_sampler = SequentialSampler(
dataset) if args.local_rank == -1 else DistributedSampler(dataset)
eval_dataloader = DataLoader(dataset,
sampler=eval_sampler,
batch_size=args.batchsize)
if not training and not args.thresh_hold_finding:
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(dataset))
logger.info(" Batch size = %d", args.batchsize)
start_time = timeit.default_timer()
pred = torch.tensor([], dtype=torch.float32, device=device)
for batch in tqdm(eval_dataloader, desc="Evaluating", disable=args.silent):
model.eval()
with torch.no_grad():
imgs, masks = batch['image'], batch['mask']
imgs = imgs.to(device=device)
masks = masks.to(device=device)
masks_pred = model(imgs).squeeze(dim=1)
masks_pred = (masks_pred > args.thresh_hold).float()
pred = torch.cat((pred, masks_pred), dim=0)
if torch.cuda.is_available():
pred = pred.cpu()
pred = torch.unsqueeze(pred, -1).numpy()
if args.test_subject < 20:
original = GE3T_postprocessing(dataset.eval_mask, pred)
else:
original = Utrecht_postprocessing(dataset.eval_mask, pred)
filename_resultImage = args.output_dir + f'/{args.test_subject}.nii.gz'
sitk.WriteImage(sitk.GetImageFromArray(original), filename_resultImage)
testImage, resultImage = getImages(dataset.eval_dir, filename_resultImage)
dsc = getDSC(testImage, resultImage)
avd = getAVD(testImage, resultImage)
#h95 distance is comment out due to numeric issues with python3 with evalution script
#h95 = getHausdorff(testImage, resultImage)
recall, f1 = getLesionDetection(testImage, resultImage)
auc = getAUC(dataset.eval_dir, filename_resultImage)
if not training and not args.thresh_hold_finding:
evalTime = timeit.default_timer() - start_time
logger.info(" Evaluation done in total %f secs (%f sec per example)",
evalTime, evalTime / len(dataset))
logger.info("***** Evaluation Results *****")
logger.info(f'Result of patient {args.test_subject}')
logger.info(f'Dice {dsc} (higher is better, max=1)')
#logger.info(f'HD {h95}mm (lower is better, min=0)')
logger.info(f'AVD {avd}% (lower is better, min=0)')
logger.info(f'Lesion detection {recall} (higher is better, max=1)')
logger.info(f'Lesion F1 {f1} (higher is better, max=1)')
logger.info(f'AUC {auc} (higher is better, max=1)')
return dsc, avd, recall, f1, auc
def test_leave_one_out(patient=0,
flair=True,
t1=True,
full=True,
first5=True,
aug=True,
verbose=False):
if patient < 20: dir = 'raw/Utrecht/'
elif patient < 40: dir = 'raw/Singapore/'
else: dir = 'raw/GE3T/'
dirs = os.listdir(dir)
dirs.sort()
dir += dirs[patient % 20]
FLAIR_image = sitk.ReadImage(dir + '/pre/FLAIR.nii.gz')
T1_image = sitk.ReadImage(dir + '/pre/T1.nii.gz')
FLAIR_array = sitk.GetArrayFromImage(FLAIR_image)
T1_array = sitk.GetArrayFromImage(T1_image)
if patient < 40: imgs_test = Utrecht_preprocessing(FLAIR_array, T1_array)
else: imgs_test = GE3T_preprocessing(FLAIR_array, T1_array)
if not flair: imgs_test = imgs_test[..., 1:2].copy()
if not t1: imgs_test = imgs_test[..., 0:1].copy()
img_shape = (rows_standard, cols_standard, flair + t1)
model = get_unet(img_shape, first5)
model_path = 'models/'
#if you want to test three models ensemble, just do like this: pred = (pred_1+pred_2+pred_3)/3
model.load_weights(model_path + str(patient) + '.h5')
pred = model.predict(imgs_test, batch_size=1, verbose=verbose)
pred[pred > 0.5] = 1.
pred[pred <= 0.5] = 0.
if patient < 40: original_pred = Utrecht_postprocessing(FLAIR_array, pred)
else: original_pred = GE3T_postprocessing(FLAIR_array, pred)
filename_resultImage = model_path + str(patient) + '.nii.gz'
sitk.WriteImage(sitk.GetImageFromArray(original_pred),
filename_resultImage)
filename_testImage = os.path.join(dir + '/wmh.nii.gz')
testImage, resultImage = getImages(filename_testImage,
filename_resultImage)
dsc = getDSC(testImage, resultImage)
avd = getAVD(testImage, resultImage)
h95 = getHausdorff(testImage, resultImage)
recall, f1 = getLesionDetection(testImage, resultImage)
return dsc, h95, avd, recall, f1
def main():
# 对三组数据 copy to predict
if_copy = True
if if_copy:
inputDir = './Data_Select_pre_T2Flair_T1_mask_sort/raw_sorted'
outputDir = './Data_Select_pre_T2Flair_T1_mask_sort/raw_nopre/'
copy_data(inputDir=inputDir, outputDir=outputDir)
# data path
TEST_INDEX = 0
inputDir = 'input_dir'
inputDir = os.path.join(inputDir, str(TEST_INDEX))
outputDir = 'result'
if not os.path.exists(outputDir):
os.mkdir(outputDir)
if_test = False
if if_test:
#Read data----------------------------------------------------------------------------
# FLAIR
FLAIR_image = sitk.ReadImage(os.path.join(inputDir, 'FLAIR.nii.gz'))
FLAIR_array = sitk.GetArrayFromImage(FLAIR_image)
# T1
T1_image = sitk.ReadImage(os.path.join(inputDir, 'T1.nii.gz'))
T1_array = sitk.GetArrayFromImage(T1_image)
imgs_test = preprocessing(np.float32(FLAIR_array),
np.float32(T1_array)) # data preprocessing
# Load model , ensemble models
img_shape = (rows_standard, cols_standard, 2)
model = get_unet(img_shape)
# weights path
model_dir = './models/our/best_weights_models/'
# Load weights
model.load_weights(os.path.join(model_dir, '0_four_json.h5'))
pred_0 = model.predict(imgs_test, batch_size=1, verbose=1)
model.load_weights(os.path.join(model_dir, '1_four_json.h5'))
pred_1 = model.predict(imgs_test, batch_size=1, verbose=1)
model.load_weights(os.path.join(model_dir, '2_four_json.h5'))
pred_2 = model.predict(imgs_test, batch_size=1, verbose=1)
model.load_weights(os.path.join(model_dir, '3_four_json.h5'))
pred_3 = model.predict(imgs_test, batch_size=1, verbose=1)
# select predict model weight count
pred_model_count = 4 # 1,2,3,4
print('Predict Model Weight Count: {}'.format(pred_model_count))
if pred_model_count == 1:
pred = pred_0
elif pred_model_count == 2:
pred = (pred_0 + pred_1) / 2
elif pred_model_count == 3:
pred = (pred_0 + pred_1 + pred_2) / 3
else:
pred = (pred_0 + pred_1 + pred_2 + pred_3) / 4
pred[pred[..., 0] > 0.45] = 1 # 0.45 thresholding
pred[pred[..., 0] <= 0.45] = 0
# get the original size to match
original_pred = postprocessing(FLAIR_array, pred)
# Save predict mask data to .nii.gz
filename_resultImage = os.path.join(outputDir, 'preidict.nii.gz')
sitk.WriteImage(sitk.GetImageFromArray(original_pred),
filename_resultImage)
compute_metric = True
if compute_metric:
filename_testImage = os.path.join(inputDir + '/wmh.nii.gz')
testImage, resultImage = getImages(filename_testImage,
filename_resultImage)
dsc = getDSC(testImage, resultImage)
avd = getAVD(testImage, resultImage)
recall, f1 = getLesionDetection(testImage, resultImage)
print('Result of prediction:')
print('Dice', dsc, ('higher is better, max=1'))
print('AVD', avd, '%', '(lower is better, min=0)')
print('Lesion detection', recall, '(higher is better, max=1)')
print('Lesion F1', f1, '(higher is better, max=1)')
print('Utrecht!')
original_pred = Utrecht_postprocessing(FLAIR_array, pred)
elif np.array_equal(para_FLAIR[0], para_array[1]):
print('Singapore!')
original_pred = Utrecht_postprocessing(FLAIR_array, pred)
elif np.array_equal(para_FLAIR[0], para_array[2]):
print('GE3T!')
original_pred = GE3T_postprocessing(FLAIR_array, pred)
if not os.path.exists(outputDir):
os.mkdir(outputDir)
filename_resultImage = os.path.join(outputDir, 'result.nii.gz')
sitk.WriteImage(sitk.GetImageFromArray(original_pred),
filename_resultImage)
filename_testImage = os.path.join(inputDir, dir_name, 'wmh.nii.gz')
testImage, resultImage = getImages(filename_testImage,
filename_resultImage)
dsc = getDSC(testImage, resultImage)
avd = getAVD(testImage, resultImage)
h95 = getHausdorff(testImage, resultImage)
recall, f1 = getLesionDetection(testImage, resultImage)
print('Result of patient ' + str(patient_count))
print('Dice', dsc, '(higher is better, max=1)')
print('HD', h95, 'mm', '(lower is better, min=0)')
print('AVD', avd, '%', '(lower is better, min=0)')
print('Lesion detection', recall, '(higher is better, max=1)')
print('Lesion F1', f1, '(higher is better, max=1)')
#Save result-------------------------------------------------------
result_output_dir = os.path.join(outputDir,
dir_name) #directory for images
if not os.path.exists(result_output_dir):
os.mkdir(result_output_dir)
images = Variable(images.cuda())
predicted = model(images)
for i in range(0, len(images)):
image_to_save = np.squeeze(predicted[i].cpu().data.numpy())
predicted_array[:, :, z_count] = image_to_save
# predicted_image = np.concatenate(predicted_image,image_to_save)
z_count += 1
predicted_output = nib.Nifti1Image(predicted_array, affine=None)
subject_name = test_subject.split('\\')
nib.save(predicted_output, subject_name[-1] + '_predicted.nii')
nib.save(out_image, subject_name[-1] + '_actual.nii')
testImage, resultImage = evaluation.getImages(
subject_name[-1] + '_actual.nii', subject_name[-1] + '_predicted.nii')
dsc = evaluation.getDSC(testImage, resultImage)
avd = evaluation.getAVD(testImage, resultImage)
recall, f1 = evaluation.getLesionDetection(testImage, resultImage)
os.remove(subject_name[-1] + '_predicted.nii')
os.remove(subject_name[-1] + '_actual.nii')
evaluation_matrix.append([subject_name, dsc, avd, recall, f1])
dsc_list.append(dsc)
avd_list.append(avd)
recall_list.append(recall)
f1_list.append(f1)
print('Average Dice score for held-out test set',
for img_idx in range(0, len(images)):
image_to_save = np.squeeze(predicted[img_idx].cpu().data.numpy())
predicted_array[:, :, z_count] = image_to_save
# predicted_image = np.concatenate(predicted_image,image_to_save)
z_count += 1
predicted_output = nib.Nifti1Image(predicted_array, affine=None)
# subject_name = test_subject.split('\\')
nib.save(predicted_output, 'predicted.nii')
'''
computes accuracy metrics on predicted and actual output
this is done for each test subject in leave one out fashion
and finally aggregate score is computed
'''
testImage, resultImage = evaluation.getImages(
os.path.join('dir_output', out_images[i]), 'predicted.nii')
dsc = evaluation.getDSC(testImage, resultImage)
avd = evaluation.getAVD(testImage, resultImage)
recall, f1 = evaluation.getLesionDetection(testImage, resultImage)
os.remove('predicted.nii')
evaluation_matrix.append(
[out_images[i].split('_')[0], dsc, avd, recall, f1])
dsc_list.append(dsc)
avd_list.append(avd)
recall_list.append(recall)
f1_list.append(f1)
'''
the average score is saved in this file for all test subjects