def predict_test(config, PCA_Net, model_name, test_loader,
patches_imgs_test, new_height, new_width, stride_height,
stride_width, full_img_height, full_img_width,
angle):
best_last = config.get('testing settings', 'best_last')
# Load the saved model
model = PCA_Net
model.load_state_dict(torch.load(path_experiment + model_name, map_location='cpu'))
model.cuda()
# Predict test data
predictions = np.empty(patches_imgs_test.shape)
for idx, (imgs, masks) in tqdm.tqdm(enumerate(test_loader)):
imgs = imgs.cuda()
mini_batch = imgs.size()[0]
outputs = model(imgs)
predictions[idx * batch_size:idx * batch_size + mini_batch, :, :, :] = outputs.cpu().detach().numpy()
print("predicted images size :")
print(predictions.shape)
pred_patches = predictions
# new_height, new_width 为DRIVE_test_imgs_original的height和width
pred_imgs = recompone_overlap(pred_patches, new_height, new_width, stride_height, stride_width) # predictions
print("pred imgs shape: " + str(pred_imgs.shape))
## back to original dimensions
pred_imgs = pred_imgs[:,:,0:full_img_height,0:full_img_width]
print("pred imgs shape: " +str(pred_imgs.shape)) # [20,1,584,584]
print("rotate angle:" + str(angle*90))
pred_imgs = np.rot90(pred_imgs, angle, (2, 3)) # np.rot90(m,k,(x,y))
pred_imgs_back = pred_imgs[:,:,0:584,0:565]
print("pred imgs shape: " + str(pred_imgs_back.shape))
return pred_imgs_back
model.load_weights(path_experiment + name_experiment + '_' + best_last +
'_weights.h5')
# Calculate the predictions
predictions = model.predict(patches_imgs_test, batch_size=32, verbose=2)
print("predicted images size :")
print((predictions.shape))
# ===== Convert the prediction arrays in corresponding images
pred_patches = pred_to_imgs(predictions, patch_height, patch_width, "original")
# ========== Elaborate and visualize the predicted images ====================
pred_imgs = None
orig_imgs = None
gtruth_masks = None
if average_mode == True:
pred_imgs = recompone_overlap(pred_patches, new_height, new_width,
stride_height, stride_width) # predictions
orig_imgs = my_PreProc(
test_imgs_orig[0:pred_imgs.shape[0], :, :, :]) # originals
gtruth_masks = masks_test # ground truth masks
else:
pred_imgs = recompone(pred_patches, 13, 12) # predictions
orig_imgs = recompone(patches_imgs_test, 13, 12) # originals
gtruth_masks = recompone(patches_masks_test, 13, 12) # masks
# apply the DRIVE masks on the repdictions #set everything outside the FOV to zero!!
kill_border(pred_imgs,
test_border_masks) # DRIVE MASK #only for visualization
## back to original dimensions
orig_imgs = orig_imgs[:, :, 0:full_img_height, 0:full_img_width]
pred_imgs = pred_imgs[:, :, 0:full_img_height, 0:full_img_width]
gtruth_masks = gtruth_masks[:, :, 0:full_img_height, 0:full_img_width]
print(("Orig imgs shape: " + str(orig_imgs.shape)))
def testing(channels, height, width, img_dir, borderMasks_dir, path_data = "./dataset_training/"):
img = np.empty((height, width, channels))
border_mask = np.empty((height, width))
if not os.path.exists(path_data):
os.makedirs(path_data)
#with path, subdirs, file in os.walk(self.img_dir):
img = Image.open(img_dir)
b_mask = Image.open(borderMasks_dir)
#border_mask = np.reshape(border_mask,(1,height,width))
assert(np.max(border_masks)==255)
assert(np.min(border_masks)==0)
"""
img = np.transpose(img,(0,3,1,2))
assert(img.shape == (channels,height,width))
border_mask = np.reshape(border_masks,(1,height,width))
assert(border_mask.shape == (1,height,width))
"""
print ("saving train datasets")
write_hdf5(img, path_data + "img.hdf5")
write_hdf5(border_mask, path_data + "borderMask.hdf5")
#original test images (for FOV selection)
DRIVE_test_imgs_original = path_data + 'img.hdf5'
test_imgs_orig = load_hdf5(DRIVE_test_imgs_original)
full_img_height = test_imgs_orig.shape[2]
full_img_width = test_imgs_orig.shape[3]
#the border masks provided by the DRIVE
DRIVE_test_border_masks = path_data + 'borderMask.hdf5'
test_border_masks = load_hdf5(DRIVE_test_border_masks)
# dimension of the patches
patch_height = 64
patch_width = 64
#the stride in case output with average
stride_height = 5
stride_width = 5
assert (stride_height < patch_height and stride_width < patch_width)
#model name
name_experiment = 'test'
path_experiment = './' + name_experiment +'/'
#N full images to be predicted
Imgs_to_test = 2
#Grouping of the predicted images
N_visual = 1
#====== average mode ===========
average_mode = True
#============ Load the data and divide in patches
patches_imgs_test = None
new_height = None
new_width = None
masks_test = None
patches_masks_test = None
if average_mode == True:
patches_imgs_test, new_height, new_width, masks_test = get_data_testing_overlap(
DRIVE_test_imgs_original = DRIVE_test_imgs_original, #original
DRIVE_test_groudTruth = path_data + 'DRIVE_dataset_groundTruth_test.hdf5', #masks
Imgs_to_test = 20,
patch_height = patch_height,
patch_width = patch_width,
stride_height = stride_height,
stride_width = stride_width
)
else:
patches_imgs_test, patches_masks_test = get_data_testing(
DRIVE_test_imgs_original = DRIVE_test_imgs_original, #original
DRIVE_test_groudTruth = path_data + 'DRIVE_dataset_groundTruth_test.hdf5', #masks
Imgs_to_test = 20,
patch_height = patch_height,
patch_width = patch_width,
)
#================ Run the prediction of the patches ==================================
best_last = 'best'
patches_imgs_test = np.einsum('klij->kijl', patches_imgs_test)
model = M.BCDU_net_D3(input_size = (64,64,1))
model.summary()
model.load_weights('weight_lstm.hdf5')
predictions = model.predict(patches_imgs_test, batch_size=16, verbose=1)
predictions = np.einsum('kijl->klij', predictions)
print(patches_imgs_test.shape)
pred_patches = predictions
print ("predicted images size :")
print (predictions.shape)
#===== Convert the prediction arrays in corresponding images
#========== Elaborate and visualize the predicted images ====================
pred_imgs = None
orig_imgs = None
gtruth_masks = None
if average_mode == True:
pred_imgs = recompone_overlap(pred_patches, new_height, new_width, stride_height, stride_width)# predictions
orig_imgs = my_PreProc(test_imgs_orig[0:pred_imgs.shape[0],:,:,:]) #originals
gtruth_masks = masks_test #ground truth masks
else:
pred_imgs = recompone(pred_patches,13,12) # predictions
orig_imgs = recompone(patches_imgs_test,13,12) # originals
gtruth_masks = recompone(patches_masks_test,13,12) #masks
# apply the DRIVE masks on the repdictions #set everything outside the FOV to zero!!
print('killing border')
kill_border(pred_imgs, test_border_masks) #DRIVE MASK #only for visualization
## back to original dimensions
orig_imgs = orig_imgs[:,:,0:full_img_height,0:full_img_width]
pred_imgs = pred_imgs[:,:,0:full_img_height,0:full_img_width]
gtruth_masks = gtruth_masks[:,:,0:full_img_height,0:full_img_width]
np.save('pred_imgs',pred_imgs)
print ("Orig imgs shape: " +str(orig_imgs.shape))
print ("pred imgs shape: " +str(pred_imgs.shape))
print ("Gtruth imgs shape: " +str(gtruth_masks.shape))
np.save('resutls', pred_imgs)
np.save('origin', gtruth_masks)
assert (orig_imgs.shape[0]==pred_imgs.shape[0] and orig_imgs.shape[0]==gtruth_masks.shape[0])
N_predicted = orig_imgs.shape[0]
group = N_visual
assert (N_predicted%group==0)
#====== Evaluate the results
print ("\n\n======== Evaluate the results =======================")
#predictions only inside the FOV
y_scores, y_true = pred_only_FOV(pred_imgs,gtruth_masks, test_border_masks) #returns data only inside the FOV
print(y_scores.shape)
print ("Calculating results only inside the FOV:")
print ("y scores pixels: " +str(y_scores.shape[0]) +" (radius 270: 270*270*3.14==228906), including background around retina: " +str(pred_imgs.shape[0]*pred_imgs.shape[2]*pred_imgs.shape[3]) +" (584*565==329960)")
print ("y true pixels: " +str(y_true.shape[0]) +" (radius 270: 270*270*3.14==228906), including background around retina: " +str(gtruth_masks.shape[2]*gtruth_masks.shape[3]*gtruth_masks.shape[0])+" (584*565==329960)")
#Area under the ROC curve
fpr, tpr, thresholds = roc_curve((y_true), y_scores)
AUC_ROC = roc_auc_score(y_true, y_scores)
# test_integral = np.trapz(tpr,fpr) #trapz is numpy integration
print ("\nArea under the ROC curve: " +str(AUC_ROC))
roc_curve =plt.figure()
plt.plot(fpr,tpr,'-',label='Area Under the Curve (AUC = %0.4f)' % AUC_ROC)
plt.title('ROC curve')
plt.xlabel("FPR (False Positive Rate)")
plt.ylabel("TPR (True Positive Rate)")
plt.legend(loc="lower right")
plt.savefig(path_experiment+"ROC.png")
#Precision-recall curve
precision, recall, thresholds = precision_recall_curve(y_true, y_scores)
precision = np.fliplr([precision])[0] #so the array is increasing (you won't get negative AUC)
recall = np.fliplr([recall])[0] #so the array is increasing (you won't get negative AUC)
AUC_prec_rec = np.trapz(precision,recall)
print ("\nArea under Precision-Recall curve: " +str(AUC_prec_rec))
prec_rec_curve = plt.figure()
plt.plot(recall,precision,'-',label='Area Under the Curve (AUC = %0.4f)' % AUC_prec_rec)
plt.title('Precision - Recall curve')
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.legend(loc="lower right")
plt.savefig(path_experiment+"Precision_recall.png")
#Confusion matrix
threshold_confusion = 0.5
print ("\nConfusion matrix: Custom threshold (for positive) of " +str(threshold_confusion))
y_pred = np.empty((y_scores.shape[0]))
for i in range(y_scores.shape[0]):
if y_scores[i]>=threshold_confusion:
y_pred[i]=1
else:
y_pred[i]=0
confusion = confusion_matrix(y_true, y_pred)
print (confusion)
accuracy = 0
if float(np.sum(confusion))!=0:
accuracy = float(confusion[0,0]+confusion[1,1])/float(np.sum(confusion))
print ("Global Accuracy: " +str(accuracy))
specificity = 0
if float(confusion[0,0]+confusion[0,1])!=0:
specificity = float(confusion[0,0])/float(confusion[0,0]+confusion[0,1])
print ("Specificity: " +str(specificity))
sensitivity = 0
if float(confusion[1,1]+confusion[1,0])!=0:
sensitivity = float(confusion[1,1])/float(confusion[1,1]+confusion[1,0])
print ("Sensitivity: " +str(sensitivity))
precision = 0
if float(confusion[1,1]+confusion[0,1])!=0:
precision = float(confusion[1,1])/float(confusion[1,1]+confusion[0,1])
print ("Precision: " +str(precision))
#Jaccard similarity index
jaccard_index = jaccard_similarity_score(y_true, y_pred, normalize=True)
print ("\nJaccard similarity score: " +str(jaccard_index))
#F1 score
F1_score = f1_score(y_true, y_pred, labels=None, average='binary', sample_weight=None)
print ("\nF1 score (F-measure): " +str(F1_score))
#Save the results
file_perf = open(path_experiment+'performances.txt', 'w')
file_perf.write("Area under the ROC curve: "+str(AUC_ROC)
+ "\nArea under Precision-Recall curve: " +str(AUC_prec_rec)
+ "\nJaccard similarity score: " +str(jaccard_index)
+ "\nF1 score (F-measure): " +str(F1_score)
+"\n\nConfusion matrix:"
+str(confusion)
+"\nACCURACY: " +str(accuracy)
+"\nSENSITIVITY: " +str(sensitivity)
+"\nSPECIFICITY: " +str(specificity)
+"\nPRECISION: " +str(precision)
)
file_perf.close()
# Visualize
fig,ax = plt.subplots(10,3,figsize=[15,15])
for idx in range(10):
ax[idx, 0].imshow(np.uint8(np.squeeze((orig_imgs[idx]))))
ax[idx, 1].imshow(np.squeeze(gtruth_masks[idx]), cmap='gray')
ax[idx, 2].imshow(np.squeeze(pred_imgs[idx]), cmap='gray')
plt.savefig(path_experiment+'sample_results.png')
'_' + best_last + '_weights.h5')
#Calculate the predictions
predictions = model.predict(patches_imgs_test, batch_size=32, verbose=2)
print "predicted images size :"
print predictions.shape
#===== Convert the prediction arrays in corresponding images
pred_patches = pred_to_imgs(predictions, patch_height, patch_width, "original")
pred_imgs = None
orig_imgs = None
gtruth_masks = None
if average_mode == True:
pred_imgs = recompone_overlap(pred_patches, full_img_height,
full_img_width, stride_h,
stride_w) # predictions
orig_imgs = my_PreProc(
test_imgs_original[0:pred_imgs.shape[0], :, :, :]) #originals
gtruth_masks = test_masks #ground truth masks
else:
pred_imgs = recompone(pred_patches, 10, 10) # predictions
orig_imgs = recompone(patches_imgs_test, 10, 10) # originals
gtruth_masks = recompone(patches_masks_test, 10, 10) #masks
# apply the DRIVE masks on the repdictions #set everything outside the FOV to zero!!
#kill_border(pred_imgs, test_border_masks) #DRIVE MASK #only for visualization
## back to original dimensions
orig_imgs = orig_imgs[:, :, 0:full_img_height, 0:full_img_width]
pred_imgs = pred_imgs[:, :, 0:full_img_height, 0:full_img_width]
gtruth_masks = gtruth_masks[:, :, 0:full_img_height, 0:full_img_width]
def predict_and_save(list_input_images, model):
print("-------------------------------------")
print("Found " + str(len(list_input_images)) + " images in directory " +
PATH_INPUT + "\n")
print(
"It should take minutes per image, depending on your CPU capacity. You can go out and come back later ;)\n"
)
for input_image in list_input_images:
start_time_image = time.time()
print("Working on " + input_image)
img_test = get_image(PATH_INPUT + input_image)
### get image sizes
# test_imgs_orig = img_test
full_img_height = img_test.shape[2]
full_img_width = img_test.shape[3]
print("Size: " + str(full_img_height) + "x" + str(full_img_width))
### Images to patches:
patches_imgs_test = None
new_height = None
new_width = None
patches_imgs_test, new_height, new_width = get_data(
imgs_test=img_test,
patch_height=patch_height,
patch_width=patch_width,
stride_height=stride_height,
stride_width=stride_width)
### Calculate the predictions
print("Computing output...")
predictions = model.predict(patches_imgs_test,
batch_size=32,
verbose=2)
# print ( "predicted images size :")
# print ( predictions.shape)
### Patches back to image
pred_patches = pred_to_imgs(predictions, patch_height, patch_width,
"original")
pred_imgs = None
pred_imgs = recompone_overlap(pred_patches, new_height, new_width,
stride_height, stride_width)
pred_imgs = pred_imgs[:, :, 0:full_img_height, 0:full_img_width]
assert (pred_imgs.shape[0] == 1)
# N_predicted = pred_imgs.shape[0]
# group = 1
# Save predictions to files
# for i in range(int(N_predicted)):
pred_stripe = group_images(pred_imgs[:, :, :, :], 1)
# file_name = input_image
visualize(pred_stripe,
"output/" + input_image[0:len(input_image) - 4] + "_pred",
mode="jpg")
elapsed_time_image = time.time() - start_time_image
print("Time consumed: " + str(int(elapsed_time_image)) + "s\n")
stride_w = int(config.get('testing settings', 'stride_width'))
test_imgs = paint_border_overlap(imgs, patch_h, patch_w, stride_h,
stride_w)
new_h, new_w = test_imgs.shape[2], test_imgs.shape[3]
for i in range(test_imgs.shape[0]):
test_img = test_imgs[i]
patches_img = extract_ordered_overlap(test_img[np.newaxis,
...], patch_h, patch_w,
stride_h, stride_w)
predictions = model.predict(patches_img, batch_size=32, verbose=2)
pred_patches = pred_to_imgs(predictions, patch_h, patch_w, "original")
pred_img = recompone_overlap(pred_patches, new_h, new_w, stride_h,
stride_w)
pred_img = pred_img[0, :, 0:614, 0:921]
img_name = img_names[i]
orig_img = np.array(
Image.open(os.path.join(path_test_ims, img_name)).convert('RGB'))
img_name = img_name.split('.')
pred_img = hard_pred(pred_img)
#pred_img = smooth_pred(pred_img)
pred_img = np.transpose(pred_img * 255, (1, 2, 0)).astype(np.uint8)
pred_image = Image.fromarray(pred_img, 'RGB').resize(
(orig_img.shape[1], orig_img.shape[0]), Image.NEAREST)
pred_image.save(
os.path.join(path_out_ims, img_name[0] + '_pred.' + img_name[1]))
for i in range(batches):
if i % 50 == 0:
print(str(i) + ' / ' + str(batches))
batch_gt_np = tf.reshape(batch_gt[:, 1],
(batch_size, 1, patch_size[0], patch_size[1]))
batch_img_np, batch_gt_np = session.run([batch_img, batch_gt_np])
patches_embedding[i * batch_size:i * batch_size +
batch_size] = batch_img_np
patches_embedding_gt[i * batch_size:i * batch_size +
batch_size] = batch_gt_np
patches_embedding = (patches_embedding[:n_samples] + 3.) / 6.
patches_embedding_gt = patches_embedding_gt[:n_samples]
orig_imgs = recompone_overlap(patches_embedding, full_img_height,
full_img_width, stride_size[0],
stride_size[1]) * 255
gtruth_masks = recompone_overlap(patches_embedding_gt, full_img_height,
full_img_width, stride_size[0],
stride_size[1]) * 255
#================ Run the prediction of the patches ==================================
best_last = config.get('testing settings', 'best_last')
#Load the saved model
if u_net:
model = get_unet(1,
batch_size,
patch_size[0],
patch_size[1],
with_activation=True) #the U-net model
def test(experiment_path, test_epoch):
# ========= CONFIG FILE TO READ FROM =======
config = configparser.RawConfigParser()
config.read('./' + experiment_path + '/' + experiment_path + '_config.txt')
# ===========================================
# run the training on invariant or local
path_data = config.get('data paths', 'path_local')
model = config.get('training settings', 'model')
# original test images (for FOV selection)
DRIVE_test_imgs_original = path_data + config.get('data paths', 'test_imgs_original')
test_imgs_orig = load_hdf5(DRIVE_test_imgs_original)
full_img_height = test_imgs_orig.shape[2]
full_img_width = test_imgs_orig.shape[3]
# the border masks provided by the DRIVE
DRIVE_test_border_masks = path_data + config.get('data paths', 'test_border_masks')
test_border_masks = load_hdf5(DRIVE_test_border_masks)
# dimension of the patches
patch_height = int(config.get('data attributes', 'patch_height'))
patch_width = int(config.get('data attributes', 'patch_width'))
# the stride in case output with average
stride_height = int(config.get('testing settings', 'stride_height'))
stride_width = int(config.get('testing settings', 'stride_width'))
assert (stride_height < patch_height and stride_width < patch_width)
# model name
name_experiment = config.get('experiment name', 'name')
path_experiment = './' + name_experiment + '/'
# N full images to be predicted
Imgs_to_test = int(config.get('testing settings', 'full_images_to_test'))
# Grouping of the predicted images
N_visual = int(config.get('testing settings', 'N_group_visual'))
# ====== average mode ===========
average_mode = config.getboolean('testing settings', 'average_mode')
#N_subimgs = int(config.get('training settings', 'N_subimgs'))
#batch_size = int(config.get('training settings', 'batch_size'))
#epoch_size = N_subimgs // (batch_size)
# #ground truth
# gtruth= path_data + config.get('data paths', 'test_groundTruth')
# img_truth= load_hdf5(gtruth)
# visualize(group_images(test_imgs_orig[0:20,:,:,:],5),'original')#.show()
# visualize(group_images(test_border_masks[0:20,:,:,:],5),'borders')#.show()
# visualize(group_images(img_truth[0:20,:,:,:],5),'gtruth')#.show()
# ============ Load the data and divide in patches
patches_imgs_test = None
new_height = None
new_width = None
masks_test = None
patches_masks_test = None
if average_mode == True:
patches_imgs_test, new_height, new_width, masks_test= get_data_testing_overlap(
DRIVE_test_imgs_original = DRIVE_test_imgs_original, #original'DRIVE_datasets_training_testing/test_hard_masks.npy'
DRIVE_test_groudTruth = path_data + config.get('data paths', 'test_groundTruth'), #masks
Imgs_to_test = int(config.get('testing settings', 'full_images_to_test')),
patch_height = patch_height,
patch_width = patch_width,
stride_height = stride_height,
stride_width = stride_width)
else:
patches_imgs_test, patches_masks_test = get_data_testing_test(
DRIVE_test_imgs_original = DRIVE_test_imgs_original, #original
DRIVE_test_groudTruth = path_data + config.get('data paths', 'test_groundTruth'), #masks
Imgs_to_test = int(config.get('testing settings', 'full_images_to_test')),
patch_height = patch_height,
patch_width = patch_width
)
#np.save(path_experiment + 'test_patches.npy', patches_imgs_test)
#visualize(group_images(patches_imgs_test,100),'./'+name_experiment+'/'+"test_patches")
# ================ Run the prediction of the patches ==================================
best_last = config.get('testing settings', 'best_last')
# Load the saved model
if model == 'UNet':
net = UNet(n_channels=1, n_classes=2)
elif model == 'UNet_cat':
net = UNet_cat(n_channels=1, n_classes=2)
else:
net = UNet_level4_our(n_channels=1, n_classes=2)
# load data
test_data = data.TensorDataset(torch.tensor(patches_imgs_test),torch.zeros(patches_imgs_test.shape[0]))
test_loader = data.DataLoader(test_data, batch_size=1, pin_memory=True, shuffle=False)
trained_model = path_experiment + 'DRIVE_' + str(test_epoch) + 'epoch.pth'
print(trained_model)
# trained_model= path_experiment+'DRIVE_unet2_B'+str(60*epoch_size)+'.pth'
net.load_state_dict(torch.load(trained_model))
net.eval()
print('Finished loading model :' + trained_model)
net = net.cuda()
cudnn.benchmark = True
# Calculate the predictions
predictions_out = np.empty((patches_imgs_test.shape[0],patch_height*patch_width,2))
for i_batch, (images, targets) in enumerate(test_loader):
images = Variable(images.float().cuda())
out1= net(images)
pred = out1.permute(0,2,3,1)
pred = F.softmax(pred, dim=-1)
pred = pred.data.view(-1,patch_height*patch_width,2)
predictions_out[i_batch] = pred
# ===== Convert the prediction arrays in corresponding images
pred_patches_out = pred_to_imgs(predictions_out, patch_height, patch_width, "original")
#np.save(path_experiment + 'pred_patches_' + str(test_epoch) + "_epoch" + '.npy', pred_patches_out)
#visualize(group_images(pred_patches_out,100),'./'+name_experiment+'/'+"pred_patches")
#========== Elaborate and visualize the predicted images ====================
pred_imgs_out = None
orig_imgs = None
gtruth_masks = None
if average_mode == True:
pred_imgs_out = recompone_overlap(pred_patches_out,new_height,new_width, stride_height, stride_width)
orig_imgs = my_PreProc(test_imgs_orig[0:pred_imgs_out.shape[0],:,:,:]) #originals
gtruth_masks = masks_test #ground truth masks
else:
pred_imgs_out = recompone(pred_patches_out,10,9) # predictions
orig_imgs = recompone(patches_imgs_test,10,9) # originals
gtruth_masks = recompone(patches_masks_test,10,9) #masks
# apply the DRIVE masks on the repdictions #set everything outside the FOV to zero!!
# DRIVE MASK #only for visualization
kill_border(pred_imgs_out, test_border_masks)
# back to original dimensions
orig_imgs = orig_imgs[:,:,0:full_img_height,0:full_img_width]
pred_imgs_out = pred_imgs_out[:, :, 0:full_img_height, 0:full_img_width]
gtruth_masks = gtruth_masks[:, :, 0:full_img_height, 0:full_img_width]
print ("Orig imgs shape: "+str(orig_imgs.shape))
print("pred imgs shape: " + str(pred_imgs_out.shape))
print("Gtruth imgs shape: " + str(gtruth_masks.shape))
np.save(path_experiment + 'pred_img_' + str(test_epoch) + "_epoch" + '.npy',pred_imgs_out)
# visualize(group_images(orig_imgs,N_visual),path_experiment+"all_originals")#.show()
if average_mode == True:
visualize(group_images(pred_imgs_out, N_visual),
path_experiment + "all_predictions_" + str(test_epoch) + "thresh_epoch")
else:
visualize(group_images(pred_imgs_out, N_visual),
path_experiment + "all_predictions_" + str(test_epoch) + "epoch_no_average")
visualize(group_images(gtruth_masks, N_visual), path_experiment + "all_groundTruths")
# visualize results comparing mask and prediction:
# assert (orig_imgs.shape[0] == pred_imgs_out.shape[0] and orig_imgs.shape[0] == gtruth_masks.shape[0])
# N_predicted = orig_imgs.shape[0]
# group = N_visual
# assert (N_predicted%group == 0)
# ====== Evaluate the results
print("\n\n======== Evaluate the results =======================")
# predictions only inside the FOV
y_scores, y_true = pred_only_FOV(pred_imgs_out, gtruth_masks, test_border_masks) # returns data only inside the FOV
'''
print("Calculating results only inside the FOV:")
print("y scores pixels: " + str(
y_scores.shape[0]) + " (radius 270: 270*270*3.14==228906), including background around retina: " + str(
pred_imgs_out.shape[0] * pred_imgs_out.shape[2] * pred_imgs_out.shape[3]) + " (584*565==329960)")
print("y true pixels: " + str(
y_true.shape[0]) + " (radius 270: 270*270*3.14==228906), including background around retina: " + str(
gtruth_masks.shape[2] * gtruth_masks.shape[3] * gtruth_masks.shape[0]) + " (584*565==329960)")
'''
# Area under the ROC curve
fpr, tpr, thresholds = roc_curve((y_true), y_scores)
AUC_ROC = roc_auc_score(y_true, y_scores)
# test_integral = np.trapz(tpr,fpr) #trapz is numpy integration
print("\nArea under the ROC curve: " + str(AUC_ROC))
rOc_curve = plt.figure()
plt.plot(fpr, tpr, '-', label='Area Under the Curve (AUC = %0.4f)' % AUC_ROC)
plt.title('ROC curve')
plt.xlabel("FPR (False Positive Rate)")
plt.ylabel("TPR (True Positive Rate)")
plt.legend(loc="lower right")
plt.savefig(path_experiment + "ROC.png")
# Precision-recall curve
precision, recall, thresholds = precision_recall_curve(y_true, y_scores)
precision = np.fliplr([precision])[0] # so the array is increasing (you won't get negative AUC)
recall = np.fliplr([recall])[0] # so the array is increasing (you won't get negative AUC)
AUC_prec_rec = np.trapz(precision, recall)
print("\nArea under Precision-Recall curve: " + str(AUC_prec_rec))
prec_rec_curve = plt.figure()
plt.plot(recall, precision, '-', label='Area Under the Curve (AUC = %0.4f)' % AUC_prec_rec)
plt.title('Precision - Recall curve')
plt.xlabel("Recall")
plt.ylabel("Precision")
plt.legend(loc="lower right")
plt.savefig(path_experiment + "Precision_recall.png")
# Confusion matrix
threshold_confusion = 0.5
print("\nConfusion matrix: Custom threshold (for positive) of " + str(threshold_confusion))
y_pred = np.empty((y_scores.shape[0]))
for i in range(y_scores.shape[0]):
if y_scores[i] >= threshold_confusion:
y_pred[i] = 1
else:
y_pred[i] = 0
confusion = confusion_matrix(y_true, y_pred)
print(confusion)
accuracy = 0
if float(np.sum(confusion)) != 0:
accuracy = float(confusion[0, 0] + confusion[1, 1]) / float(np.sum(confusion))
print("Global Accuracy: " + str(accuracy))
specificity = 0
if float(confusion[0, 0] + confusion[0, 1]) != 0:
specificity = float(confusion[0, 0]) / float(confusion[0, 0] + confusion[0, 1])
print("Specificity: " + str(specificity))
sensitivity = 0
if float(confusion[1, 1] + confusion[1, 0]) != 0:
sensitivity = float(confusion[1, 1]) / float(confusion[1, 1] + confusion[1, 0])
print("Sensitivity: " + str(sensitivity))
precision = 0
if float(confusion[1, 1] + confusion[0, 1]) != 0:
precision = float(confusion[1, 1]) / float(confusion[1, 1] + confusion[0, 1])
print("Precision: " + str(precision))
# Jaccard similarity index
jaccard_index = jaccard_similarity_score(y_true, y_pred, normalize=True)
print("\nJaccard similarity score: " + str(jaccard_index))
# F1 score
F1_score = f1_score(y_true, y_pred, labels=None, average='binary', sample_weight=None)
print("\nF1 score (F-measure): " + str(F1_score))
####evaluate the thin vessels
thin_3pixel_recall_indivi = []
thin_3pixel_auc_roc = []
for j in range(pred_imgs_out.shape[0]):
thick3=opening(gtruth_masks[j, 0, :, :], square(3))
thin_gt = gtruth_masks[j, 0, :, :] - thick3
thin_pred=pred_imgs_out[j, 0, :, :]
thin_pred[thick3==1]=0
thin_3pixel_recall_indivi.append(round(thin_recall(thin_gt, pred_imgs_out[j, 0, :, :], thresh=0.5), 4))
thin_3pixel_auc_roc.append(round(roc_auc_score(thin_gt.flatten(), thin_pred.flatten()), 4))
thin_2pixel_recall_indivi = []
thin_2pixel_auc_roc = []
for j in range(pred_imgs_out.shape[0]):
thick=opening(gtruth_masks[j, 0, :, :], square(2))
thin_gt = gtruth_masks[j, 0, :, :] - thick
#thin_gt_only=thin_gt[thin_gt==1]
#print(thin_gt_only)
thin_pred=pred_imgs_out[j, 0, :, :]
#thin_pred=thin_pred[thin_gt==1]
thin_pred[thick==1]=0
thin_2pixel_recall_indivi.append(round(thin_recall(thin_gt, pred_imgs_out[j, 0, :, :], thresh=0.5), 4))
thin_2pixel_auc_roc.append(round(roc_auc_score(thin_gt.flatten(), thin_pred.flatten()), 4))
#print("thin 2vessel recall:", thin_2pixel_recall_indivi)
#print('thin 2vessel auc score', thin_2pixel_auc_roc)
# Save the results
with open(path_experiment + 'test_performances_all_epochs.txt', mode='a') as f:
f.write("\n\n" + path_experiment + " test epoch:" + str(test_epoch)
+ '\naverage mode is:' + str(average_mode)
+ "\nArea under the ROC curve: %.4f" % (AUC_ROC)
+ "\nArea under Precision-Recall curve: %.4f" % (AUC_prec_rec)
+ "\nJaccard similarity score: %.4f" % (jaccard_index)
+ "\nF1 score (F-measure): %.4f" % (F1_score)
+ "\nConfusion matrix:"
+ str(confusion)
+ "\nACCURACY: %.4f" % (accuracy)
+ "\nSENSITIVITY: %.4f" % (sensitivity)
+ "\nSPECIFICITY: %.4f" % (specificity)
+ "\nPRECISION: %.4f" % (precision)
+ "\nthin 2vessels recall indivi:\n" + str(thin_2pixel_recall_indivi)
+ "\nthin 2vessels recall mean:%.4f" % (np.mean(thin_2pixel_recall_indivi))
+ "\nthin 2vessels auc indivi:\n" + str(thin_2pixel_auc_roc)
+ "\nthin 2vessels auc score mean:%.4f" % (np.mean(thin_2pixel_auc_roc))
+ "\nthin 3vessels recall indivi:\n" + str(thin_3pixel_recall_indivi)
+ "\nthin 3vessels recall mean:%.4f" % (np.mean(thin_3pixel_recall_indivi))
+ "\nthin 3vessels auc indivi:\n" + str(thin_3pixel_auc_roc)
+ "\nthin 3vessels auc score mean:%.4f" % (np.mean(thin_3pixel_auc_roc))
)
#Calculate the predictions
predictions = model.predict(patches_imgs_test, batch_size=32, verbose=2)
print "predicted images size :"
print predictions.shape
#===== Convert the prediction arrays in corresponding images
pred_patches = pred_to_imgs(predictions,"original")
#========== Elaborate and visualize the predicted images ====================
pred_imgs = None
orig_imgs = None
gtruth_masks = None
if average_mode == True:
pred_imgs = recompone_overlap(pred_patches, new_height, new_width, stride_height, stride_width)# predictions
orig_imgs = my_PreProc(test_imgs_orig[0:pred_imgs.shape[0],:,:,:]) #originals
gtruth_masks = masks_test #ground truth masks
else:
pred_imgs = recompone(pred_patches,13,12) # predictions
orig_imgs = recompone(patches_imgs_test,13,12) # originals
gtruth_masks = recompone(patches_masks_test,13,12) #masks
# apply the DRIVE masks on the repdictions #set everything outside the FOV to zero!!
kill_border(pred_imgs, test_border_masks) #DRIVE MASK #only for visualization
## back to original dimensions
orig_imgs = orig_imgs[:,:,0:full_img_height,0:full_img_width]
pred_imgs = pred_imgs[:,:,0:full_img_height,0:full_img_width]
gtruth_masks = gtruth_masks[:,:,0:full_img_height,0:full_img_width]
print "Orig imgs shape: " +str(orig_imgs.shape)
print "pred imgs shape: " +str(pred_imgs.shape)
print "Gtruth imgs shape: " +str(gtruth_masks.shape)