def model_test(use_existing):
print('-' * 30)
print('Loading test data...')
print('-' * 30)
# Loading test data:
filename = cm.filename
modelname = cm.modellist[0]
originFile_list = sorted(
glob(cm.workingPath.originTestingSet_path + filename))
maskFile_list = sorted(glob(cm.workingPath.maskTestingSet_path + filename))
preFile_list = sorted(glob(cm.workingPath.testingSet_path + 'mask*.dcm'))
out_test_images = []
out_test_masks = []
out_pre_masks = []
for i in range(len(originFile_list)):
# originTestVolInfo = loadFileInformation(originFile_list[i])
# maskTestVolInfo = loadFileInformation(maskFile_list[i])
originTestVol, originTestVol_num, originTestVolwidth, originTestVolheight = loadFile(
originFile_list[i])
maskTestVol, maskTestVol_num, maskTestVolwidth, maskTestVolheight = loadFile(
maskFile_list[i])
preTestVol, preTestVol_num, preTestVolwidth, preTestVolheight = loadFile(
preFile_list[i])
for j in range(len(maskTestVol)):
maskTestVol[j] = np.where(maskTestVol[j] != 0, 1, 0)
for img in originTestVol:
out_test_images.append(img)
for img in maskTestVol:
out_test_masks.append(img)
for img in preTestVol:
out_pre_masks.append(img)
num_test_images = len(out_test_images)
final_test_images = np.ndarray([num_test_images, 512, 512], dtype=np.int16)
final_test_masks = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
final_pre_masks = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
for i in range(num_test_images):
final_test_images[i] = out_test_images[i]
final_test_masks[i] = out_test_masks[i]
final_pre_masks[i] = out_pre_masks[i]
final_test_images = np.expand_dims(final_test_images, axis=-1)
final_test_masks = np.expand_dims(final_test_masks, axis=-1)
final_pre_masks = np.expand_dims(final_pre_masks, axis=-1)
for i in range(num_test_images):
dice_coef_slice = nw.dice_coef_np(final_test_masks[i],
final_pre_masks[i])
total_progress = i / num_test_images * 100
print('predicting slice: %03d' % (i),
' total progress: %5.2f%%' % (total_progress),
' dice coef: %5.3f' % (dice_coef_slice))
def model_test(use_existing):
print('-' * 30)
print('Loading test data...')
print('-' * 30)
# Loading test data:
filename = cm.filename
modelname = cm.modellist[0]
originFile_list = sorted(
glob(cm.workingPath.originTestingSet_path + filename))
maskFile_list = sorted(glob(cm.workingPath.maskTestingSet_path + filename))
file = str(originFile_list[0])[23:]
out_test_images = []
out_test_masks = []
for i in range(len(originFile_list)):
# originTestVolInfo = loadFileInformation(originFile_list[i])
# maskTestVolInfo = loadFileInformation(maskFile_list[i])
originTestVol, originTestVol_num, originTestVolwidth, originTestVolheight = loadFile(
originFile_list[i])
maskTestVol, maskTestVol_num, maskTestVolwidth, maskTestVolheight = loadFile(
maskFile_list[i])
for j in range(len(maskTestVol)):
maskTestVol[j] = np.where(maskTestVol[j] != 0, 1, 0)
for img in originTestVol:
# new_img = resize(img, [512, 512])
out_test_images.append(img)
for img in maskTestVol:
# new_mask = resize(img, [512, 512])
out_test_masks.append(img)
num_test_images = len(out_test_images)
# num_test_images = 1
# test_num = 241
final_test_images = np.ndarray([num_test_images, 512, 512], dtype=np.int16)
final_test_masks = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
# import pdb; pdb.set_trace()
for i in range(num_test_images):
final_test_images[i] = out_test_images[i]
final_test_masks[i] = out_test_masks[i]
final_test_images = np.expand_dims(final_test_images, axis=-1)
final_test_masks = np.expand_dims(final_test_masks, axis=-1)
learning_rate = 0.0001
decay_rate = 5e-6
momentum = 0.9
# sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)
adam = Adam(lr=learning_rate, decay=decay_rate)
opti = adam
print('_' * 30)
print('calculating model...')
print('_' * 30)
# model = nw.get_unet_more_feature()
# model = nw.get_unet_less_feature()
# model = nw.get_unet_dilated_conv_4()
model = nw.get_unet(opti)
# model = nw.get_shallow_unet()
# model = nw.get_dropout_unet()
using_start_end = 0
start_slice = 121
end_slice = 283
if use_existing:
model.load_weights(modelname)
imgs_mask_test = np.ndarray([num_test_images, 512, 512, 1],
dtype=np.float32)
# logs = []
for i in range(num_test_images):
imgs_mask_test[i] = resize(
(model.predict([final_test_images[i:i + 1]], verbose=0)[0]),
[512, 512, 1])
# imgs_mask_test[i] = np.where(imgs_mask_test[i] < 0.5, 0, 1)
# imgs_mask_test[i] = morphology.erosion(imgs_mask_test[i], np.ones([2, 2, 1]))
# imgs_mask_test[i] = morphology.dilation(imgs_mask_test[i], np.ones([5, 5, 1]))
dice_coef_slice = nw.dice_coef_np(final_test_masks[i],
imgs_mask_test[i])
total_progress = i / num_test_images * 100
print('predicting slice: %03d' % (i),
' total progress: %5.2f%%' % (total_progress),
' dice coef: %5.3f' % (dice_coef_slice))
final_test_images = final_test_images + 4000
np.save(cm.workingPath.testingSet_path + 'testImages.npy',
final_test_images)
np.save(cm.workingPath.testingSet_path + 'testMasks.npy', final_test_masks)
np.save(cm.workingPath.testingSet_path + 'masksTestPredicted.npy',
imgs_mask_test)
# Load data:
imgs_origin = np.load(cm.workingPath.testingSet_path +
'testImages.npy').astype(np.int16)
imgs_true = np.load(cm.workingPath.testingSet_path +
'testMasks.npy').astype(np.int8)
imgs_predict = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(np.float32)
imgs_predict_threshold = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(
np.float32)
imgs_origin = np.squeeze(imgs_origin, axis=-1)
imgs_true = np.squeeze(imgs_true, axis=-1)
imgs_predict = np.squeeze(imgs_predict, axis=-1)
imgs_predict_threshold = np.squeeze(imgs_predict_threshold, axis=-1)
imgs_predict_threshold = np.where(imgs_predict_threshold < 0.5, 0, 1)
if using_start_end == 1:
mean = nw.dice_coef_np(imgs_predict[start_slice:end_slice],
imgs_true[start_slice:end_slice])
else:
mean = nw.dice_coef_np(imgs_predict, imgs_true)
np.savetxt(cm.workingPath.testingSet_path + 'dicemean.txt',
np.array(mean).reshape(1, ),
fmt='%.5f')
print('-' * 30)
print('Model file:', modelname)
print('Total Dice Coeff', mean)
print('-' * 30)
# Draw the subplots of figures:
color1 = 'gray' # ***
color2 = 'viridis' # ******
# color = 'plasma' # **
# color = 'magma' # ***
# color2 = 'RdPu' # ***
# color = 'gray' # ***
# color = 'gray' # ***
transparent1 = 1.0
transparent2 = 0.5
# Slice parameters:
#############################################
# Automatically:
steps = 40
slice = range(0, len(imgs_true), steps)
plt_row = 3
plt_col = int(len(imgs_true) / steps)
plt.figure(1, figsize=(25, 12))
for i in slice:
if i == 0:
plt_num = int(i / steps) + 1
else:
plt_num = int(i / steps)
if plt_num <= plt_col:
plt.figure(1)
ax1 = plt.subplot(plt_row, plt_col, plt_num)
title = 'slice=' + str(i)
plt.title(title)
ax1.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax1.imshow(imgs_true[i, :, :], cmap=color2, alpha=transparent2)
ax2 = plt.subplot(plt_row, plt_col, plt_num + plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax2.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax2.imshow(imgs_predict[i, :, :], cmap=color2, alpha=transparent2)
ax3 = plt.subplot(plt_row, plt_col, plt_num + 2 * plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax3.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax3.imshow(imgs_predict_threshold[i, :, :],
cmap=color2,
alpha=transparent2)
else:
pass
modelname = cm.modellist[0]
imageName = re.findall(r'\d+\.?\d*', modelname)
epoch_num = int(imageName[0]) + 1
accuracy = float(
np.loadtxt(cm.workingPath.testingSet_path + 'dicemean.txt', float))
# saveName = 'epoch_' + str(epoch_num) + '_dice_' +str(accuracy) + '.png'
saveName = 'epoch_%02d_dice_%.3f.png' % (epoch_num - 1, accuracy)
plt.subplots_adjust(left=0.0,
bottom=0.05,
right=1.0,
top=0.95,
hspace=0.3,
wspace=0.3)
plt.savefig(cm.workingPath.testingSet_path + saveName)
# plt.show()
###################################
# Manually:
#
# slice = (100,150,230,250)
#
# plt.figure(2)
#
# ax1 = plt.subplot(2,4,1)
# title = 'slice=' + str(slice[0])
# plt.title(title)
# ax1.imshow(new_imgs_origin[slice[0],0,:,:],cmap=color1,alpha=transparent1)
# ax1.imshow(new_imgs_true[slice[0],0,:,:],cmap=color2,alpha=transparent2)
#
# ax2 = plt.subplot(2,4,5)
# title = 'slice=' + str(slice[0])
# plt.title(title)
# ax2.imshow(new_imgs_origin[slice[0],0,:,:],cmap=color1,alpha=transparent1)
# ax2.imshow(new_imgs_predict[slice[0],0,:,:],cmap=color2,alpha=transparent2)
#
# ax3 = plt.subplot(2,4,2)
# title = 'slice=' + str(slice[1])
# plt.title(title)
# ax3.imshow(new_imgs_origin[slice[1],0,:,:],cmap=color1,alpha=transparent1)
# ax3.imshow(new_imgs_true[slice[1],0,:,:],cmap=color2,alpha=transparent2)
#
# ax4 = plt.subplot(2,4,6)
# title = 'slice=' + str(slice[1])
# plt.title(title)
# ax4.imshow(new_imgs_origin[slice[1],0,:,:],cmap=color1,alpha=transparent1)
# ax4.imshow(new_imgs_predict[slice[1],0,:,:],cmap=color2,alpha=transparent2)
#
# ax5 = plt.subplot(2,4,3)
# title = 'slice=' + str(slice[2])
# plt.title(title)
# ax5.imshow(new_imgs_origin[slice[2],0,:,:],cmap=color1,alpha=transparent1)
# ax5.imshow(new_imgs_true[slice[2],0,:,:],cmap=color2,alpha=transparent2)
#
# ax6 = plt.subplot(2,4,7)
# title = 'slice=' + str(slice[2])
# plt.title(title)
# ax6.imshow(new_imgs_origin[slice[2],0,:,:],cmap=color1,alpha=transparent1)
# ax6.imshow(new_imgs_predict[slice[2],0,:,:],cmap=color2,alpha=transparent2)
#
# ax7 = plt.subplot(2,4,4)
# title = 'slice=' + str(slice[3])
# plt.title(title)
# ax7.imshow(new_imgs_origin[slice[3],0,:,:],cmap=color1,alpha=transparent1)
# ax7.imshow(new_imgs_true[slice[3],0,:,:],cmap=color2,alpha=transparent2)
#
# ax8 = plt.subplot(2,4,8)
# title = 'slice=' + str(slice[3])
# plt.title(title)
# ax8.imshow(new_imgs_origin[slice[3],0,:,:],cmap=color1,alpha=transparent1)
# ax8.imshow(new_imgs_predict[slice[3],0,:,:],cmap=color2,alpha=transparent2)
#
# plt.show()
#############################################
print('Images saved')
# Save npy as dcm files:
imgs_origin = np.uint16(imgs_origin)
imgs_true = np.uint16(imgs_true)
imgs_predict_threshold = np.uint16(imgs_predict_threshold)
new_imgs_origin_dcm = sitk.GetImageFromArray(imgs_origin)
new_imgs_true_dcm = sitk.GetImageFromArray(imgs_true)
new_imgs_predict_dcm = sitk.GetImageFromArray(imgs_predict_threshold)
sitk.WriteImage(new_imgs_predict_dcm,
cm.workingPath.testingSet_path + "Predicted_" + file)
ds1 = dicom.read_file(maskFile_list[0])
ds2 = dicom.read_file(cm.workingPath.testingSet_path + "Predicted_" + file)
ds1.PixelData = ds2.PixelData
ds1.save_as(cm.workingPath.testingSet_path + "Predicted_" + file)
print('DICOM saved')
def train_and_predict(use_existing):
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
# Choose which subset you would like to use:
i = 0
imgs_train = np.load(cm.workingPath.training3DSet_path +
'trainImages_%04d.npy' % (i)).astype(np.float32)
imgs_mask_train = np.load(cm.workingPath.training3DSet_path +
'trainMasks_%04d.npy' % (i)).astype(np.float32)
# imgs_test = np.load(cm.workingPath.trainingSet_path + 'testImages_%04d.npy'%(i)).astype(np.float32)
# imgs_mask_test_true = np.load(cm.workingPath.trainingSet_path + 'testMasks_%04d.npy'%(i)).astype(np.float32)
# Mean for data centering:
# mean= np.mean(imgs_train)
# Std for data normalization:
# std = np.std(imgs_train)
# imgs_train -= mean
# imgs_train /= std
print('_' * 30)
print('Creating and compiling model...')
print('_' * 30)
model = nw.get_3D_unet()
# model = nw.get_3D_unet_2()
# model = nw.get_simple_3D_unet()
modelname = 'model.png'
plot_model(model,
show_shapes=True,
to_file=cm.workingPath.model_path + modelname)
model.summary()
# config = model.get_config()
# print(config)
# Callbacks:
filepath = cm.workingPath.model_path + 'weights.{epoch:02d}-{loss:.5f}.hdf5'
bestfilepath = cm.workingPath.model_path + 'Best_weights.{epoch:02d}-{loss:.5f}.hdf5'
model_checkpoint = callbacks.ModelCheckpoint(filepath,
monitor='loss',
verbose=0,
save_best_only=False)
model_best_checkpoint = callbacks.ModelCheckpoint(bestfilepath,
monitor='val_loss',
verbose=0,
save_best_only=True)
# history = cm.LossHistory_Gerda(cm.workingPath.working_path)
history = nw.LossHistory()
# model_history = callbacks.TensorBoard(log_dir='./logs', histogram_freq=1, write_graph=True, write_images=True,
# embeddings_freq=1, embeddings_layer_names=None, embeddings_metadata= None)
callbacks_list = [history, model_best_checkpoint]
# Should we load existing weights?
# Set argument for call to train_and_predict to true at end of script
if use_existing:
model.load_weights('./unet.hdf5')
print('-' * 30)
print('Fitting model...')
print('-' * 30)
model.fit(imgs_train,
imgs_mask_train,
batch_size=1,
epochs=400,
verbose=1,
shuffle=True,
validation_split=0.1,
callbacks=callbacks_list)
print('training finished')
def train_and_predict(use_existing, x_train, x_val, y_train, y_val, cross, pre_lrate):
cm.mkdir(cm.workingPath.model_path)
cm.mkdir(cm.workingPath.best_model_path)
class LossHistory(callbacks.Callback):
def on_train_begin(self, logs={}):
self.losses = [1]
self.val_losses = []
self.lr = []
def on_epoch_end(self, epoch, logs={}):
self.losses.append(logs.get('loss'))
loss_file = (list(self.losses))
np.savetxt(cm.workingPath.model_path + 'loss_' + 'Val.%02d.txt' % (cross), loss_file[1:], newline='\r\n')
self.val_losses.append(logs.get('val_loss'))
val_loss_file = (list(self.val_losses))
np.savetxt(cm.workingPath.model_path + 'val_loss_' + 'Val.%02d.txt' % (cross), val_loss_file, newline='\r\n')
self.lr.append(step_decay(len(self.losses)))
print('\nLearning rate:', step_decay(len(self.losses)))
lrate_file = (list(self.lr))
np.savetxt(cm.workingPath.model_path + 'lrate_' + 'Val.%02d.txt' % (cross), lrate_file, newline='\r\n')
if cross == 0:
learning_rate = 0.0001
decay_rate = 5e-6
momentum = 0.9
else:
learning_rate = pre_lrate
decay_rate = 5e-6
momentum = 0.9
# sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)
adam = Adam(lr=learning_rate, decay=decay_rate)
opti = adam
def step_decay(losses):
if float(2 * np.sqrt(np.array(history.losses[-1]))) < 100:
if cross == 0:
lrate = 0.0001 * 1 / (1 + 0.1 * len(history.losses))
# lrate = 0.0001
momentum = 0.8
decay_rate = 2e-6
else:
lrate = pre_lrate * 1 / (1 + 0.1 * len(history.losses))
# lrate = 0.0001
momentum = 0.8
decay_rate = 2e-6
return lrate
else:
if cross == 0:
lrate = 0.0001
else:
lrate = pre_lrate
return lrate
history = LossHistory()
lrate = LearningRateScheduler(step_decay)
print('_' * 30)
print('Creating and compiling model...')
print('_' * 30)
# model = nw.get_simple_unet(opti)
# model = nw.get_shallow_unet(sgd)
model = nw.get_unet(opti)
# model = nw.get_dropout_unet()
# model = nw.get_unet_less_feature()
# model = nw.get_unet_dilated_conv_4()
# model = nw.get_unet_dilated_conv_7()
# model = nw.get_2D_Deeply_supervised_network()
modelname = 'model.png'
# plot_model(model, show_shapes=True, to_file=cm.workingPath.model_path + modelname)
model.summary()
# Callbacks:
filepath = cm.workingPath.model_path + 'Val.%02d_' % (cross) + 'weights.{epoch:02d}-{loss:.5f}.hdf5'
bestfilepath = cm.workingPath.best_model_path + 'Val.%02d_' % (cross) + 'Best_weights.{epoch:02d}-{loss:.5f}.hdf5'
unet_hdf5_path = cm.workingPath.working_path + 'unet.hdf5'
model_checkpoint = callbacks.ModelCheckpoint(filepath, monitor='loss', verbose=0, save_best_only=True)
model_best_checkpoint = callbacks.ModelCheckpoint(bestfilepath, monitor='val_loss', verbose=0, save_best_only=True)
model_best_unet_hdf5 = callbacks.ModelCheckpoint(unet_hdf5_path, monitor='val_loss', verbose=0, save_best_only=True)
history = LossHistory()
callbacks_list = [history, lrate, model_checkpoint, model_best_checkpoint, model_best_unet_hdf5]
# Should we load existing weights?
# Set argument for call to train_and_predict to true at end of script
if use_existing:
model.load_weights('./unet.hdf5')
print('-' * 30)
print('Fitting model...')
print('-' * 30)
model.fit(x_train, y_train, batch_size=4, epochs=50, verbose=1, shuffle=True,
validation_data=(x_val, y_val), callbacks=callbacks_list)
print('training finished')
def model_test(use_existing, ii):
print('-' * 30)
print('Loading test data...')
print('-' * 30)
# Loading test data:
filename = cm.filename
modelname = cm.modellist[ii]
originFile_list = glob(cm.workingPath.originTestingSet_path + filename)
maskFile_list = glob(cm.workingPath.maskTestingSet_path + filename)
out_test_images = []
out_test_masks = []
for i in range(len(originFile_list)):
# originTestVolInfo = loadFileInformation(originFile_list[i])
# maskTestVolInfo = loadFileInformation(maskFile_list[i])
originTestVol, originTestVol_num, originTestVolwidth, originTestVolheight = loadFile(
originFile_list[i])
maskTestVol, maskTestVol_num, maskTestVolwidth, maskTestVolheight = loadFile(
maskFile_list[i])
for j in range(len(maskTestVol)):
maskTestVol[j] = np.where(maskTestVol[j] != 0, 1, 0)
for img in originTestVol:
# new_img = resize(img, [512, 512])
out_test_images.append(img)
for img in maskTestVol:
# new_mask = resize(img, [512, 512])
out_test_masks.append(img)
# num_test_images = len(out_test_images)
num_test_images = 1
test_num = 140
final_test_images = np.ndarray([num_test_images, 512, 512],
dtype=np.float32)
final_test_masks = np.ndarray([num_test_images, 512, 512],
dtype=np.float32)
# import pdb; pdb.set_trace()
final_test_images[0] = out_test_images[test_num]
final_test_masks[0] = out_test_masks[test_num]
final_test_images = np.expand_dims(final_test_images, axis=-1)
final_test_masks = np.expand_dims(final_test_masks, axis=-1)
print('_' * 30)
print('calculating model...')
print('_' * 30)
model = nw.get_unet()
if use_existing:
model.load_weights(modelname)
imgs_mask_test = np.ndarray([num_test_images, 512, 512, 1],
dtype=np.float32)
imgs_mask_test[0] = resize(
(model.predict([final_test_images[0:1]], verbose=0)[0]), [512, 512, 1])
np.save(cm.workingPath.testingSet_path + 'testImages.npy',
final_test_images)
np.save(cm.workingPath.testingSet_path + 'testMasks.npy', final_test_masks)
np.save(cm.workingPath.testingSet_path + 'masksTestPredicted.npy',
imgs_mask_test)
# Save npy as dcm files:
imgs_origin = np.load(cm.workingPath.testingSet_path +
'testImages.npy').astype(np.uint16)
imgs_true = np.load(cm.workingPath.testingSet_path +
'testMasks.npy').astype(np.uint16)
imgs_predict = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(np.uint16)
# for j in range(len(imgs_true)):
# imgs_true[j] = np.where(imgs_true[j] < 0.5, 0, 1)
# for j in range(len(imgs_predict)):
# imgs_predict[j] = np.where(imgs_predict[j] < 0.5, 0, 1)
imgs_origin = np.squeeze(imgs_origin, axis=-1)
imgs_true = np.squeeze(imgs_true, axis=-1)
imgs_predict = np.squeeze(imgs_predict, axis=-1)
new_image_origin = sitk.GetImageFromArray(imgs_origin)
new_image_true = sitk.GetImageFromArray(imgs_true)
new_image_predict = sitk.GetImageFromArray(imgs_predict)
# sitk.WriteImage(new_image_origin, cm.workingPath.testingSet_path + 'testImages.dcm')
# sitk.WriteImage(new_image_true, cm.workingPath.testingSet_path + 'testMasks.dcm')
# sitk.WriteImage(new_image_predict, cm.workingPath.testingSet_path + 'masksTestPredicted.dcm')
mean = 0.0
for i in range(num_test_images):
mean += nw.dice_coef_np(final_test_masks[i], imgs_mask_test[i])
mean /= num_test_images
np.savetxt(cm.workingPath.testingSet_path + 'dicemean.txt',
np.array(mean).reshape(1, ),
fmt='%.3f')
print('model file:', modelname)
print('Mean Dice Coeff', mean)
# Load data:
imgs_origin = np.load(cm.workingPath.testingSet_path +
'testImages.npy').astype(np.float32)
imgs_true = np.load(cm.workingPath.testingSet_path +
'testMasks.npy').astype(np.float32)
imgs_predict = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(np.float32)
# Turn images into binary images from (0,1):
for i in range(len(imgs_true)):
imgs_true[i] = np.where(imgs_true[i] < 0.5, 0, 1)
for j in range(len(imgs_predict)):
imgs_predict[j] = np.where(imgs_predict[j] < 0.5, 0, 1)
# Prepare to do some operations on images, or not:
new_imgs_origin = imgs_origin
new_imgs_true = imgs_true
new_imgs_predict = imgs_predict
# for i in range(len(imgs_true)):
# new_imgs_true[len(imgs_true)-i-1] = imgs_true[i]
#
# for i in range(len(imgs_predict)):
# new_imgs_predict[len(imgs_predict)-i-1] = imgs_predict[i]
# Draw the subplots of figures:
color1 = 'gray' # ***
color2 = 'viridis' # ******
# color = 'plasma' # **
# color = 'magma' # ***
# color2 = 'RdPu' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
# color = 'gray' # ***
transparent1 = 1.0
transparent2 = 0.5
# Slice parameters:
#############################################
# Automatically:
steps = 1
slice = range(0, len(new_imgs_true), steps)
plt_row = 2
plt_col = int(len(new_imgs_true) / steps)
plt.figure(1, figsize=(20, 10))
for i in slice:
if i == 0:
plt_num = int(i / steps) + 1
else:
plt_num = int(i / steps)
if plt_num <= plt_col:
plt.figure(1)
ax1 = plt.subplot(plt_row, plt_col, plt_num)
title = 'slice=' + str(i)
plt.title(title)
ax1.imshow(new_imgs_origin[i, :, :, 0],
cmap=color1,
alpha=transparent1)
ax1.imshow(new_imgs_true[i, :, :, 0],
cmap=color2,
alpha=transparent2)
ax2 = plt.subplot(plt_row, plt_col, plt_num + plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax2.imshow(new_imgs_origin[i, :, :, 0],
cmap=color1,
alpha=transparent1)
ax2.imshow(new_imgs_predict[i, :, :, 0],
cmap=color2,
alpha=transparent2)
else:
pass
modelname = cm.modellist[ii]
imageName = re.findall(r'\d+\.?\d*', modelname)
epoch_num = int(imageName[0]) + 1
accuracy = float(
np.loadtxt(cm.workingPath.testingSet_path + 'dicemean.txt', float))
# saveName = 'epoch_' + str(epoch_num) + '_dice_' +str(accuracy) + '.png'
saveName = 'epoch_%02d_dice_%.3f.png' % (epoch_num, accuracy)
plt.savefig(cm.workingPath.testingSet_path + saveName)
# plt.show()
###################################
# Manually:
#
# slice = (100,150,230,250)
#
# plt.figure(2)
#
# ax1 = plt.subplot(2,4,1)
# title = 'slice=' + str(slice[0])
# plt.title(title)
# ax1.imshow(new_imgs_origin[slice[0],0,:,:],cmap=color1,alpha=transparent1)
# ax1.imshow(new_imgs_true[slice[0],0,:,:],cmap=color2,alpha=transparent2)
#
# ax2 = plt.subplot(2,4,5)
# title = 'slice=' + str(slice[0])
# plt.title(title)
# ax2.imshow(new_imgs_origin[slice[0],0,:,:],cmap=color1,alpha=transparent1)
# ax2.imshow(new_imgs_predict[slice[0],0,:,:],cmap=color2,alpha=transparent2)
#
# ax3 = plt.subplot(2,4,2)
# title = 'slice=' + str(slice[1])
# plt.title(title)
# ax3.imshow(new_imgs_origin[slice[1],0,:,:],cmap=color1,alpha=transparent1)
# ax3.imshow(new_imgs_true[slice[1],0,:,:],cmap=color2,alpha=transparent2)
#
# ax4 = plt.subplot(2,4,6)
# title = 'slice=' + str(slice[1])
# plt.title(title)
# ax4.imshow(new_imgs_origin[slice[1],0,:,:],cmap=color1,alpha=transparent1)
# ax4.imshow(new_imgs_predict[slice[1],0,:,:],cmap=color2,alpha=transparent2)
#
# ax5 = plt.subplot(2,4,3)
# title = 'slice=' + str(slice[2])
# plt.title(title)
# ax5.imshow(new_imgs_origin[slice[2],0,:,:],cmap=color1,alpha=transparent1)
# ax5.imshow(new_imgs_true[slice[2],0,:,:],cmap=color2,alpha=transparent2)
#
# ax6 = plt.subplot(2,4,7)
# title = 'slice=' + str(slice[2])
# plt.title(title)
# ax6.imshow(new_imgs_origin[slice[2],0,:,:],cmap=color1,alpha=transparent1)
# ax6.imshow(new_imgs_predict[slice[2],0,:,:],cmap=color2,alpha=transparent2)
#
# ax7 = plt.subplot(2,4,4)
# title = 'slice=' + str(slice[3])
# plt.title(title)
# ax7.imshow(new_imgs_origin[slice[3],0,:,:],cmap=color1,alpha=transparent1)
# ax7.imshow(new_imgs_true[slice[3],0,:,:],cmap=color2,alpha=transparent2)
#
# ax8 = plt.subplot(2,4,8)
# title = 'slice=' + str(slice[3])
# plt.title(title)
# ax8.imshow(new_imgs_origin[slice[3],0,:,:],cmap=color1,alpha=transparent1)
# ax8.imshow(new_imgs_predict[slice[3],0,:,:],cmap=color2,alpha=transparent2)
#
# plt.show()
#############################################
print('Images showing')
def main():
""" Builds, trains, and runs the neural network. """
app_time = timer.Timer()
# Gets the phenotype names and the regression y values.
print()
pheno_names, pheno_data = dh.get_data("./data/8.pheno.2.txt", False)
print("Phenotypes:", pheno_data.shape, "\n\n", pheno_names, "\n\n",
pheno_data, "\n")
# Gets the snp names and the regressors.
snp_names, snp_data = dh.get_data("./data/8.snps.txt", True)
snp_names = np.transpose(snp_names)
snp_data = np.transpose(snp_data)
print("SNPs:", snp_data.shape, "\n\n", snp_names, "\n\n", snp_data, "\n")
# Make batches out of snp_data and unallocate snp_data
snp_data = bb.make_batches(snp_data, len(pheno_data[0]))
# Initialize graph structure.
layer = net.ConnectedLayer(len(snp_data[0][0]), OUT_SIZE)
layer.shape()
# Start TensorFlow session.
sess = sh.start_session()
past_loss = 0
step = 1
index = np.arange(len(snp_data))
while True:
# rng_state = np.random.get_state()
# np.random.shuffle(snp_data)
# np.random.set_state(rng_state)
# np.random.shuffle(pheno_data[0])
# Train for an epoch
# Get the current loss and train the graph.
np.random.shuffle(index)
for i in range(len(snp_data)):
snp_sample = snp_data[index[i]]
pheno_sample = [[pheno_data[0][index[i]]]]
current_loss, _ = sess.run([layer.loss, layer.train_step],
feed_dict={
layer.input: snp_sample,
layer.observed: pheno_sample
})
prog.progress(i, len(snp_data),
"Training Completed in Epoch " + str(step))
prog.log_training(past_loss, current_loss, ALPHA, step, app_time)
# Save the weight and bias tensors when the model converges.
if abs(past_loss - current_loss) < (ALPHA):
np.savetxt("w.out",
sess.run(layer.weight,
feed_dict={
layer.input:
snp_data[0],
layer.observed:
np.asarray([pheno_data[0][0]
]).reshape(1, OUT_SIZE)
}),
delimiter="\t")
np.savetxt("b.out",
sess.run(layer.bias,
feed_dict={
layer.input:
snp_data[0],
layer.observed:
np.asarray([pheno_data[0][0]
]).reshape(1, OUT_SIZE)
}),
delimiter="\t")
break
past_loss = current_loss
step += 1
print(" [", app_time.get_time(), "]", "Closing session...\n")
sess.close()
def model_test(use_existing):
print('-' * 30)
print('Loading test data...')
# Loading test data:
filename = cm.filename
modelname = cm.modellist[0]
originFile_list = sorted(
glob(cm.workingPath.originTestingSet_path + filename))
maskFile_list = sorted(glob(cm.workingPath.maskTestingSet_path + filename))
out_test_images = []
out_test_masks = []
for i in range(len(originFile_list)):
# originTestVolInfo = loadFileInformation(originFile_list[i])
# maskTestVolInfo = loadFileInformation(maskFile_list[i])
originTestVol, originTestVol_num, originTestVolwidth, originTestVolheight = loadFile(
originFile_list[i])
maskTestVol, maskTestVol_num, maskTestVolwidth, maskTestVolheight = loadFile(
maskFile_list[i])
for j in range(len(maskTestVol)):
maskTestVol[j] = np.where(maskTestVol[j] != 0, 1, 0)
for img in originTestVol:
out_test_images.append(img)
for img in maskTestVol:
out_test_masks.append(img)
num_test_images = len(out_test_images)
final_test_images = np.ndarray([num_test_images, 512, 512], dtype=np.int16)
final_test_masks = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
for i in range(num_test_images):
final_test_images[i] = out_test_images[i]
final_test_masks[i] = out_test_masks[i]
final_test_images = np.expand_dims(final_test_images, axis=-1)
final_test_masks = np.expand_dims(final_test_masks, axis=-1)
row = nw.img_rows_3d
col = nw.img_cols_3d
num_rowes = 1
num_coles = 1
row_1 = int((512 - row) / 2)
row_2 = int(512 - (512 - row) / 2)
col_1 = int((512 - col) / 2)
col_2 = int(512 - (512 - col) / 2)
slices = nw.slices_3d
gaps = nw.gaps_3d
final_images_crop = final_test_images[:, row_1:row_2, col_1:col_2, :]
final_masks_crop = final_test_masks[:, row_1:row_2, col_1:col_2, :]
num_patches = int((num_test_images - slices) / gaps)
num_patches1 = int(final_images_crop.shape[0] / slices)
test_image = np.ndarray([1, slices, row, col, 1], dtype=np.int16)
test_mask = np.ndarray([1, slices, row, col, 1], dtype=np.int8)
predicted_mask_volume = np.ndarray([num_test_images, row, col],
dtype=np.float32)
# model = nw.get_3D_unet()
model = nw.get_3D_CNN()
# model = nw.get_3D_unet_drop_1()
# model = nw.get_3D_unet_BN()
using_start_end = 1
start_slice = cm.start_slice
end_slice = -1
if use_existing:
model.load_weights(modelname)
for i in range(num_patches):
count1 = i * gaps
count2 = i * gaps + slices
test_image[0] = final_images_crop[count1:count2]
test_mask[0] = final_masks_crop[count1:count2]
predicted_mask = model.predict(test_image)
predicted_mask_volume[count1:count2] += predicted_mask[0, :, :, :, 0]
predicted_mask_volume = np.expand_dims(predicted_mask_volume, axis=-1)
np.save(cm.workingPath.testingSet_path + 'testImages.npy',
final_images_crop)
np.save(cm.workingPath.testingSet_path + 'testMasks.npy', final_masks_crop)
np.save(cm.workingPath.testingSet_path + 'masksTestPredicted.npy',
predicted_mask_volume)
imgs_origin = np.load(cm.workingPath.testingSet_path +
'testImages.npy').astype(np.int16)
imgs_true = np.load(cm.workingPath.testingSet_path +
'testMasks.npy').astype(np.int8)
imgs_predict = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(np.float32)
imgs_origin = np.squeeze(imgs_origin, axis=-1)
imgs_true = np.squeeze(imgs_true, axis=-1)
imgs_predict = np.squeeze(imgs_predict, axis=-1)
imgs_predict_threshold = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(
np.float32)
imgs_predict_threshold = np.squeeze(imgs_predict_threshold, axis=-1)
imgs_predict_threshold = np.where(imgs_predict_threshold < (8), 0, 1)
if using_start_end == 1:
mean = nw.dice_coef_np(imgs_predict_threshold[start_slice:end_slice],
imgs_true[start_slice:end_slice])
else:
mean = nw.dice_coef_np(imgs_predict_threshold, imgs_true)
np.savetxt(cm.workingPath.testingSet_path + 'dicemean.txt',
np.array(mean).reshape(1, ),
fmt='%.5f')
print('Model file:', modelname)
print('Total Dice Coeff', mean)
print('-' * 30)
# mean=[]
#
# for thre in range(1,17):
# imgs_predict_threshold = np.load(cm.workingPath.testingSet_path + 'masksTestPredicted.npy').astype(np.float32)
# imgs_predict_threshold = np.squeeze(imgs_predict_threshold, axis=-1)
# imgs_predict_threshold = np.where(imgs_predict_threshold < (thre), 0, 1)
#
# if using_start_end == 1:
# meantemp = nw.dice_coef_np(imgs_predict_threshold[start_slice:end_slice], imgs_true[start_slice:end_slice])
# mean.append(meantemp)
# else:
# meantemp = nw.dice_coef_np(imgs_predict_threshold, imgs_true)
# mean.append(meantemp)
#
# np.savetxt(cm.workingPath.testingSet_path + 'dicemean.txt', mean, fmt='%.5f')
#
# print('Model file:', modelname)
# print('Total Dice Coeff', mean)
# print('-' * 30)
# Draw the subplots of figures:
color1 = 'gray' # ***
color2 = 'viridis' # ******
# color = 'plasma' # **
# color = 'magma' # ***
# color2 = 'RdPu' # ***
# color = 'gray' # ***
# color = 'gray' # ***
transparent1 = 1.0
transparent2 = 0.5
# Slice parameters:
#############################################
# Automatically:
steps = 40
slice = range(0, len(imgs_origin), steps)
plt_row = 3
plt_col = int(len(imgs_origin) / steps)
plt.figure(1, figsize=(25, 12))
for i in slice:
if i == 0:
plt_num = int(i / steps) + 1
else:
plt_num = int(i / steps)
if plt_num <= plt_col:
plt.figure(1)
ax1 = plt.subplot(plt_row, plt_col, plt_num)
title = 'slice=' + str(i)
plt.title(title)
ax1.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax1.imshow(imgs_true[i, :, :], cmap=color2, alpha=transparent2)
ax2 = plt.subplot(plt_row, plt_col, plt_num + plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax2.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax2.imshow(imgs_predict[i, :, :], cmap=color2, alpha=transparent2)
ax3 = plt.subplot(plt_row, plt_col, plt_num + 2 * plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax3.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax3.imshow(imgs_predict_threshold[i, :, :],
cmap=color2,
alpha=transparent2)
else:
pass
modelname = cm.modellist[0]
imageName = re.findall(r'\d+\.?\d*', modelname)
epoch_num = int(imageName[0]) + 1
accuracy = float(
np.loadtxt(cm.workingPath.testingSet_path + 'dicemean.txt', float))
# saveName = 'epoch_' + str(epoch_num) + '_dice_' +str(accuracy) + '.png'
saveName = 'epoch_%02d_dice_%.3f.png' % (epoch_num - 1, accuracy)
plt.subplots_adjust(left=0.0,
bottom=0.05,
right=1.0,
top=0.95,
hspace=0.3,
wspace=0.3)
plt.savefig(cm.workingPath.testingSet_path + saveName)
# plt.show()
print('Images saved')
# Save npy as dcm files:
# final_test_predicted_threshold = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
# final_test_images = np.squeeze(final_test_images + 4000, axis=-1)
final_test_masks = np.squeeze(final_test_masks, axis=-1)
# final_test_images[0:num_patches1 * slices, row_1:row_2, col_1:col_2,] = imgs_origin[:, :, :]
# final_test_masks[0:num_patches1 * slices:, row_1:row_2, col_1:col_2,] = imgs_true[:, :, :]
final_test_predicted_threshold = final_test_masks
final_test_predicted_threshold[:, row_1:row_2, col_1:
col_2] = imgs_predict_threshold[:, :, :]
final_test_predicted_threshold = np.uint16(final_test_predicted_threshold)
new_imgs_predict_dcm = sitk.GetImageFromArray(
final_test_predicted_threshold)
sitk.WriteImage(new_imgs_predict_dcm,
cm.workingPath.testingSet_path + 'masksTestPredicted.dcm')
ds1 = dicom.read_file(maskFile_list[0])
ds2 = dicom.read_file(cm.workingPath.testingSet_path +
'masksTestPredicted.dcm')
ds1.PixelData = ds2.PixelData
ds1.save_as(cm.workingPath.testingSet_path + 'masksTestPredicted.dcm')
print('DICOM saved')
def model_test(use_existing, modelnum):
print('-' * 30)
print('Loading test data...')
print('-' * 30)
# Loading test data:
filename = cm.filename
modelname = cm.modelname
originFile_list = glob(cm.workingPath.originTestingSet_path + filename)
maskFile_list = glob(cm.workingPath.maskTestingSet_path + filename)
out_test_images = []
out_test_masks = []
for i in range(len(originFile_list)):
# originTestVolInfo = loadFileInformation(originFile_list[i])
# maskTestVolInfo = loadFileInformation(maskFile_list[i])
originTestVol, originTestVol_num, originTestVolwidth, originTestVolheight = loadFile(
originFile_list[i])
maskTestVol, maskTestVol_num, maskTestVolwidth, maskTestVolheight = loadFile(
maskFile_list[i])
for j in range(len(maskTestVol)):
maskTestVol[j] = np.where(maskTestVol[j] != 0, 1, 0)
for img in originTestVol:
# new_img = resize(img, [512, 512])
out_test_images.append(img)
for img in maskTestVol:
# new_mask = resize(img, [512, 512])
out_test_masks.append(img)
# num_test_images = len(out_test_images)
num_test_images = 1
test_num = 177
final_test_images = np.ndarray([num_test_images, 512, 512],
dtype=np.float32)
final_test_masks = np.ndarray([num_test_images, 512, 512],
dtype=np.float32)
# import pdb; pdb.set_trace()
for i in range(num_test_images):
final_test_images[i] = out_test_images[i + test_num]
final_test_masks[i] = out_test_masks[i + test_num]
final_test_images = np.expand_dims(final_test_images, axis=-1)
final_test_masks = np.expand_dims(final_test_masks, axis=-1)
print('_' * 30)
print('calculating model...')
print('_' * 30)
# model = nw.get_unet_more_feature()
model = nw.get_unet()
if use_existing:
model.load_weights(cm.modellist[modelnum])
imgs_mask_test = np.ndarray([num_test_images, 512, 512, 1],
dtype=np.float32)
# logs = []
for i in range(num_test_images):
imgs_mask_test[i] = resize(
(model.predict([final_test_images[i:i + 1]], verbose=0)[0]),
[512, 512, 1])
# imgs_mask_test[i] = np.where(imgs_mask_test[i] < 0.5, 0, 1)
# imgs_mask_test[i] = morphology.erosion(imgs_mask_test[i], np.ones([2, 2, 1]))
# imgs_mask_test[i] = morphology.dilation(imgs_mask_test[i], np.ones([5, 5, 1]))
dice_coef_slice = nw.dice_coef_np(final_test_masks[i],
imgs_mask_test[i])
total_progress = i / num_test_images * 100
# log_slice = 'predicting slice: %03d' %(i),' total progress: %5.2f%%' %(total_progress), ' dice coef: %5.3f'%(dice_coef_slice)
# log_slice = str(log_slice)
# np.append(logs, log_slice)
print('predicting slice: %03d' % (i),
' total progress: %5.2f%%' % (total_progress),
' dice coef: %5.3f' % (dice_coef_slice))
# logs = np.array(logs)
# np.savetxt(cm.workingPath.testingSet_path + 'logs.txt', logs)
final_test_images = final_test_images + 4000
np.save(cm.workingPath.testingSet_path + 'testImages.npy',
final_test_images)
np.save(cm.workingPath.testingSet_path + 'testMasks.npy', final_test_masks)
np.save(cm.workingPath.testingSet_path + 'masksTestPredicted.npy',
imgs_mask_test)
# Load data:
imgs_origin = np.load(cm.workingPath.testingSet_path +
'testImages.npy').astype(np.float32)
imgs_true = np.load(cm.workingPath.testingSet_path +
'testMasks.npy').astype(np.float32)
imgs_predict = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(np.float32)
imgs_predict_threshold = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(
np.float32)
imgs_origin = np.squeeze(imgs_origin, axis=-1)
imgs_true = np.squeeze(imgs_true, axis=-1)
imgs_predict = np.squeeze(imgs_predict, axis=-1)
imgs_predict_threshold = np.squeeze(imgs_predict_threshold, axis=-1)
imgs_predict_threshold = np.where(imgs_predict_threshold < 0.5, 0, 1)
mean = nw.dice_coef_np(imgs_predict, imgs_true)
np.savetxt(cm.workingPath.testingSet_path + 'dicemean.txt',
np.array(mean).reshape(1, ),
fmt='%.5f')
print('-' * 30)
print('Model file:', modelname)
print('Total Dice Coeff', mean)
print('-' * 30)
# Draw the subplots of figures:
color1 = 'gray' # ***
color2 = 'viridis' # ******
# color = 'plasma' # **
# color = 'magma' # ***
# color2 = 'RdPu' # ***
# color = 'gray' # ***
# color = 'gray' # ***
transparent1 = 1.0
transparent2 = 0.5
# Slice parameters:
#############################################
# Automatically:
steps = 1
slice = range(0, len(imgs_true), steps)
plt_row = 3
plt_col = int(len(imgs_true) / steps)
plt.figure('Single Slice', figsize=(4, 12))
for i in slice:
if i == 0:
plt_num = int(i / steps) + 1
else:
plt_num = int(i / steps)
if plt_num <= plt_col:
plt.figure(1)
ax1 = plt.subplot(plt_row, plt_col, plt_num)
title = 'slice=' + str(i)
plt.title(title)
ax1.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax1.imshow(imgs_true[i, :, :], cmap=color2, alpha=transparent2)
ax2 = plt.subplot(plt_row, plt_col, plt_num + plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax2.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax2.imshow(imgs_predict[i, :, :], cmap=color2, alpha=transparent2)
ax3 = plt.subplot(plt_row, plt_col, plt_num + 2 * plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax3.imshow(imgs_origin[i, :, :], cmap=color1, alpha=transparent1)
ax3.imshow(imgs_predict_threshold[i, :, :],
cmap=color2,
alpha=transparent2)
else:
pass
modelname = cm.modelname
imageName = re.findall(r'\d+\.?\d*', modelname)
epoch_num = int(imageName[0]) + 1
accuracy = float(
np.loadtxt(cm.workingPath.testingSet_path + 'dicemean.txt', float))
# saveName = 'epoch_' + str(epoch_num) + '_dice_' +str(accuracy) + '.png'
saveName = 'epoch_%02d_dice_%.3f.png' % (modelnum, accuracy)
plt.subplots_adjust(left=0.0,
bottom=0.05,
right=1.0,
top=0.95,
hspace=0.3,
wspace=0.3)
plt.savefig(cm.workingPath.testingSet_path + saveName)
# plt.show()
###################################
# Manually:
#
# slice = (100,150,230,250)
#
# plt.figure(2)
#
# ax1 = plt.subplot(2,4,1)
# title = 'slice=' + str(slice[0])
# plt.title(title)
# ax1.imshow(new_imgs_origin[slice[0],0,:,:],cmap=color1,alpha=transparent1)
# ax1.imshow(new_imgs_true[slice[0],0,:,:],cmap=color2,alpha=transparent2)
#
# ax2 = plt.subplot(2,4,5)
# title = 'slice=' + str(slice[0])
# plt.title(title)
# ax2.imshow(new_imgs_origin[slice[0],0,:,:],cmap=color1,alpha=transparent1)
# ax2.imshow(new_imgs_predict[slice[0],0,:,:],cmap=color2,alpha=transparent2)
#
# ax3 = plt.subplot(2,4,2)
# title = 'slice=' + str(slice[1])
# plt.title(title)
# ax3.imshow(new_imgs_origin[slice[1],0,:,:],cmap=color1,alpha=transparent1)
# ax3.imshow(new_imgs_true[slice[1],0,:,:],cmap=color2,alpha=transparent2)
#
# ax4 = plt.subplot(2,4,6)
# title = 'slice=' + str(slice[1])
# plt.title(title)
# ax4.imshow(new_imgs_origin[slice[1],0,:,:],cmap=color1,alpha=transparent1)
# ax4.imshow(new_imgs_predict[slice[1],0,:,:],cmap=color2,alpha=transparent2)
#
# ax5 = plt.subplot(2,4,3)
# title = 'slice=' + str(slice[2])
# plt.title(title)
# ax5.imshow(new_imgs_origin[slice[2],0,:,:],cmap=color1,alpha=transparent1)
# ax5.imshow(new_imgs_true[slice[2],0,:,:],cmap=color2,alpha=transparent2)
#
# ax6 = plt.subplot(2,4,7)
# title = 'slice=' + str(slice[2])
# plt.title(title)
# ax6.imshow(new_imgs_origin[slice[2],0,:,:],cmap=color1,alpha=transparent1)
# ax6.imshow(new_imgs_predict[slice[2],0,:,:],cmap=color2,alpha=transparent2)
#
# ax7 = plt.subplot(2,4,4)
# title = 'slice=' + str(slice[3])
# plt.title(title)
# ax7.imshow(new_imgs_origin[slice[3],0,:,:],cmap=color1,alpha=transparent1)
# ax7.imshow(new_imgs_true[slice[3],0,:,:],cmap=color2,alpha=transparent2)
#
# ax8 = plt.subplot(2,4,8)
# title = 'slice=' + str(slice[3])
# plt.title(title)
# ax8.imshow(new_imgs_origin[slice[3],0,:,:],cmap=color1,alpha=transparent1)
# ax8.imshow(new_imgs_predict[slice[3],0,:,:],cmap=color2,alpha=transparent2)
#
# plt.show()
#############################################
print('Images saved')
def model_test(use_existing):
print('-' * 30)
print('Loading test data...')
print('-' * 30)
# Loading test data:
filename = cm.filename
modelname = cm.modellist[0]
originFile_list = sorted(
glob(cm.workingPath.originTestingSet_path + filename))
maskFile_list = sorted(glob(cm.workingPath.maskTestingSet_path + filename))
out_test_images = []
out_test_masks = []
for i in range(len(originFile_list)):
# originTestVolInfo = loadFileInformation(originFile_list[i])
# maskTestVolInfo = loadFileInformation(maskFile_list[i])
originTestVol, originTestVol_num, originTestVolwidth, originTestVolheight = loadFile(
originFile_list[i])
maskTestVol, maskTestVol_num, maskTestVolwidth, maskTestVolheight = loadFile(
maskFile_list[i])
for j in range(len(maskTestVol)):
maskTestVol[j] = np.where(maskTestVol[j] != 0, 1, 0)
for img in originTestVol:
out_test_images.append(img)
for img in maskTestVol:
out_test_masks.append(img)
num_test_images = len(out_test_images)
final_test_images = np.ndarray([num_test_images, 512, 512], dtype=np.int16)
final_test_masks = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
# import pdb; pdb.set_trace()
for i in range(num_test_images):
final_test_images[i] = out_test_images[i]
final_test_masks[i] = out_test_masks[i]
final_test_images = np.expand_dims(final_test_images, axis=-1)
final_test_masks = np.expand_dims(final_test_masks, axis=-1)
row = nw.img_rows_3d
col = nw.img_cols_3d
num_rows = 3
num_coles = 3
row_1 = int((512 - row * num_rows) / 2)
row_2 = int(512 - (512 - row * num_rows) / 2)
col_1 = int((512 - col * num_coles) / 2)
col_2 = int(512 - (512 - col * num_coles) / 2)
slices = 32
final_test_images_crop = final_test_images[:, row_1:row_2, col_1:col_2, :]
final_test_masks_crop = final_test_masks[:, row_1:row_2, col_1:col_2, :]
imgs_image_test = np.ndarray([0, row, col, 1], dtype=np.int16)
imgs_mask_test = np.ndarray([0, row, col, 1], dtype=np.float32)
num_patches = int(final_test_images_crop.shape[0] / slices)
for num_patch in range(0, num_patches):
for num_row in range(0, num_rows):
for num_col in range(0, num_coles):
imgs_image_test = np.concatenate(
(imgs_image_test,
final_test_images_crop[num_patch *
slices:(num_patch * slices +
slices), num_row *
row:(num_row * row + row),
num_col * col:(num_col * col +
col), :]),
axis=0)
imgs_mask_test = np.concatenate(
(imgs_mask_test,
final_test_masks_crop[num_patch *
slices:(num_patch * slices +
slices), num_row *
row:(num_row * row + row), num_col *
col:(num_col * col + col), :]),
axis=0)
tubes = int(imgs_image_test.shape[0] / slices)
imgs_image_test_last = np.ndarray([tubes, slices, row, col, 1],
dtype=np.int16)
imgs_mask_test_last = np.ndarray([tubes, slices, row, col, 1],
dtype=np.int8)
imgs_predict_last = np.ndarray([tubes, slices, row, col, 1],
dtype=np.float32)
for i in range(tubes):
imgs_image_test_last[i, :, :, :, :] = imgs_image_test[i *
slices:(i + 1) *
slices, :, :, :]
imgs_mask_test_last[i, :, :, :, :] = imgs_mask_test[i *
slices:(i + 1) *
slices, :, :, :]
print('_' * 30)
print('calculating model...')
print('_' * 30)
model = nw.get_3D_unet()
if use_existing:
model.load_weights(modelname)
# logs = []
for i in range(imgs_image_test_last.shape[0]):
imgs_predict_last[i] = model.predict([imgs_image_test_last[i:i + 1]])
# imgs_mask_test[i] = np.where(imgs_mask_test[i] < 0.5, 0, 1)
# imgs_mask_test[i] = morphology.erosion(imgs_mask_test[i], np.ones([2, 2, 1]))
# imgs_mask_test[i] = morphology.dilation(imgs_mask_test[i], np.ones([5, 5, 1]))
dice_coef_slice = nw.dice_coef_np(imgs_predict_last[i],
imgs_mask_test_last[i])
total_progress = i / imgs_image_test_last.shape[0] * 100
print('predicting slice: %03d' % (i),
' total progress: %5.2f%%' % (total_progress),
' dice coef: %5.3f' % (dice_coef_slice))
imgs_image_test_last = imgs_image_test_last + 4000
np.save(cm.workingPath.testingSet_path + 'testImages.npy',
imgs_image_test_last)
np.save(cm.workingPath.testingSet_path + 'testMasks.npy',
imgs_mask_test_last)
np.save(cm.workingPath.testingSet_path + 'masksTestPredicted.npy',
imgs_predict_last)
# Load data:
imgs_origin = np.load(cm.workingPath.testingSet_path +
'testImages.npy').astype(np.int16)
imgs_true = np.load(cm.workingPath.testingSet_path +
'testMasks.npy').astype(np.int8)
imgs_predict = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(np.float32)
imgs_predict_threshold = np.load(cm.workingPath.testingSet_path +
'masksTestPredicted.npy').astype(
np.float32)
imgs_origin = np.squeeze(imgs_origin, axis=-1)
imgs_true = np.squeeze(imgs_true, axis=-1)
imgs_predict = np.squeeze(imgs_predict, axis=-1)
imgs_predict_threshold = np.squeeze(imgs_predict_threshold, axis=-1)
imgs_predict_threshold = np.where(imgs_predict_threshold < 0.5, 0, 1)
mean = nw.dice_coef_np(imgs_predict_threshold, imgs_true)
np.savetxt(cm.workingPath.testingSet_path + 'dicemean.txt',
np.array(mean).reshape(1, ),
fmt='%.5f')
print('-' * 30)
print('Model file:', modelname)
print('Total Dice Coeff', mean)
print('-' * 30)
imgs_image_sized = np.ndarray(
[num_patches * slices, row * num_rows, col * num_coles],
dtype=np.int16)
imgs_mask_sized = np.ndarray(
[num_patches * slices, row * num_rows, col * num_coles], dtype=np.int8)
imgs_predict_sized = np.ndarray(
[num_patches * slices, row * num_rows, col * num_coles],
dtype=np.float32)
imgs_predict_threshold_sized = np.ndarray(
[num_patches * slices, row * num_rows, col * num_coles], dtype=np.int8)
num_patches = int(final_test_images_crop.shape[0] / slices)
for num_patch in range(0, num_patches):
for num_row in range(0, num_rows):
for num_col in range(0, num_coles):
count = num_patch * num_rows * num_coles + num_row * num_coles + num_col
imgs_image_sized[num_patch * slices:(num_patch + 1) * slices,
num_row * row:(num_row + 1) * row,
num_col * col:(num_col + 1) *
col] = imgs_origin[count, :, :, :]
imgs_mask_sized[num_patch * slices:(num_patch + 1) * slices,
num_row * row:(num_row + 1) * row,
num_col * col:(num_col + 1) *
col] = imgs_true[count, :, :, :]
imgs_predict_sized[num_patch * slices:(num_patch + 1) * slices,
num_row * row:(num_row + 1) * row,
num_col * col:(num_col + 1) *
col] = imgs_predict[count, :, :, :]
imgs_predict_threshold_sized[
num_patch * slices:(num_patch + 1) * slices,
num_row * row:(num_row + 1) * row,
num_col * col:(num_col + 1) *
col] = imgs_predict_threshold[count, :, :, :]
# Draw the subplots of figures:
color1 = 'gray' # ***
color2 = 'viridis' # ******
# color = 'plasma' # **
# color = 'magma' # ***
# color2 = 'RdPu' # ***
# color = 'gray' # ***
# color = 'gray' # ***
transparent1 = 1.0
transparent2 = 0.5
# Slice parameters:
#############################################
# Automatically:
steps = 40
slice = range(0, len(imgs_mask_sized), steps)
plt_row = 3
plt_col = int(len(imgs_mask_sized) / steps)
plt.figure(1, figsize=(25, 12))
for i in slice:
if i == 0:
plt_num = int(i / steps) + 1
else:
plt_num = int(i / steps)
if plt_num <= plt_col:
plt.figure(1)
ax1 = plt.subplot(plt_row, plt_col, plt_num)
title = 'slice=' + str(i)
plt.title(title)
ax1.imshow(imgs_image_sized[i, :, :],
cmap=color1,
alpha=transparent1)
ax1.imshow(imgs_mask_sized[i, :, :],
cmap=color2,
alpha=transparent2)
ax2 = plt.subplot(plt_row, plt_col, plt_num + plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax2.imshow(imgs_image_sized[i, :, :],
cmap=color1,
alpha=transparent1)
ax2.imshow(imgs_predict_sized[i, :, :],
cmap=color2,
alpha=transparent2)
ax3 = plt.subplot(plt_row, plt_col, plt_num + 2 * plt_col)
title = 'slice=' + str(i)
plt.title(title)
ax3.imshow(imgs_image_sized[i, :, :],
cmap=color1,
alpha=transparent1)
ax3.imshow(imgs_predict_threshold_sized[i, :, :],
cmap=color2,
alpha=transparent2)
else:
pass
modelname = cm.modellist[0]
imageName = re.findall(r'\d+\.?\d*', modelname)
epoch_num = int(imageName[0]) + 1
accuracy = float(
np.loadtxt(cm.workingPath.testingSet_path + 'dicemean.txt', float))
# saveName = 'epoch_' + str(epoch_num) + '_dice_' +str(accuracy) + '.png'
saveName = 'epoch_%02d_dice_%.3f.png' % (epoch_num, accuracy)
plt.subplots_adjust(left=0.0,
bottom=0.05,
right=1.0,
top=0.95,
hspace=0.3,
wspace=0.3)
plt.savefig(cm.workingPath.testingSet_path + saveName)
# plt.show()
print('Images saved')
# Save npy as dcm files:
# final_test_predicted_threshold = np.ndarray([num_test_images, 512, 512], dtype=np.int8)
final_test_images = np.squeeze(final_test_images + 4000, axis=-1)
final_test_masks = np.squeeze(final_test_masks, axis=-1)
final_test_images[0:num_patches * slices, row_1:row_2,
col_1:col_2] = imgs_image_sized[:, :, :]
final_test_masks[0:num_patches * slices:, row_1:row_2,
col_1:col_2] = imgs_mask_sized[:, :, :]
final_test_predicted_threshold = final_test_masks
final_test_predicted_threshold[
0:num_patches * slices, row_1:row_2,
col_1:col_2] = imgs_predict_threshold_sized[:, :, :]
final_test_images = np.uint16(final_test_images)
final_test_masks = np.uint16(final_test_masks)
final_test_predicted_threshold = np.uint16(final_test_predicted_threshold)
new_imgs_origin_dcm = sitk.GetImageFromArray(final_test_images)
new_imgs_true_dcm = sitk.GetImageFromArray(final_test_masks)
new_imgs_predict_dcm = sitk.GetImageFromArray(
final_test_predicted_threshold)
# sitk.WriteImage(new_imgs_origin_dcm, cm.workingPath.testingSet_path + 'testImages.dcm')
# sitk.WriteImage(new_imgs_true_dcm, cm.workingPath.testingSet_path + 'testMasks.dcm')
sitk.WriteImage(new_imgs_predict_dcm,
cm.workingPath.testingSet_path + 'masksTestPredicted.dcm')
ds1 = dicom.read_file(maskFile_list[0])
ds2 = dicom.read_file(cm.workingPath.testingSet_path +
'masksTestPredicted.dcm')
ds1.PixelData = ds2.PixelData
ds1.save_as(cm.workingPath.testingSet_path + 'masksTestPredicted.dcm')
print('DICOM saved')