def resample_all(new_spacing, isnormal, shift_need): # _rd = _read_data(2,train_tag='train/',validation_tag='validation/',test_tag='test/', # img_name='CT.mha',label_name='GTV_CT.mha') # _rd = _read_data(1, train_tag='prostate_train/', validation_tag='prostate_validation/', test_tag='prostate_test/', # img_name='CTImage.mha', label_name='Bladder.mha') resampled_path = '/exports/lkeb-hpc/syousefi/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Data-01/21data1100data2-v3/' _rd = _read_data(2, train_tag='train/', validation_tag='validation/', test_tag='test/', img_name='CT.mha', label_name='GTV_CT.mha', resampled_path=resampled_path) '''read path of the images for train, test, and validation''' train_CTs, train_GTVs, train_Torso, validation_CTs, validation_GTVs, validation_Torso, \ test_CTs, test_GTVs, test_Torso, depth, width, height = _rd.read_image_path() extract_torsos(train_CTs) # padding and casting: padding_images(train_CTs, train_GTVs, train_Torso, 57) padding_images(validation_CTs, validation_GTVs, validation_Torso, 57) padding_images(test_CTs, test_GTVs, test_Torso, 57)
def resample_all(new_spacing,isnormal,shift_need): # _rd = _read_data(2,train_tag='train/',validation_tag='validation/',test_tag='test/', # img_name='CT.mha',label_name='GTV_CT.mha') # _rd = _read_data(1, train_tag='prostate_train/', validation_tag='prostate_validation/', test_tag='prostate_test/', # img_name='CTImage.mha', label_name='Bladder.mha') _rd = _read_data(2, train_tag='train/', validation_tag='validation/', test_tag='Extra/', img_name='CT.mha', label_name='GTV_prim.mha') '''read path of the images for train, test, and validation''' train_CTs, train_GTVs, train_Torso, validation_CTs, validation_GTVs, validation_Torso, \ test_CTs, test_GTVs, test_Torso, depth, width, height = _rd.read_image_path() # extract_torsos(train_CTs) # padding and casting: # padding_images(train_CTs, train_GTVs,train_Torso, 57) # padding_images(validation_CTs, validation_GTVs, validation_Torso, 57) padding_images(test_CTs, test_GTVs, test_Torso, 57)
def __init__(self): self.train_tag='train/' self.validation_tag='validation/' self.test_tag='test/' self.img_name='CT_padded.mha' self.label_name='GTV_CT_padded.mha' self.torso_tag='CT_padded_Torso.mha' self.data=2 self.gtv_min_z = [] self.gtv_max_z = [] self.gtv_min_x = [] self.gtv_max_x = [] self.gtv_min_y = [] self.gtv_max_y = [] self.mask_min_z = [] self.mask_max_z = [] self.mask_min_x = [] self.mask_max_x = [] self.mask_min_y = [] self.mask_max_y = [] self._rd = _read_data(data=self.data, train_tag=self.train_tag, validation_tag=self.validation_tag, test_tag=self.test_tag, img_name=self.img_name, label_name=self.label_name, torso_tag=self.torso_tag)
from os.path import isfile, join from random import shuffle import matplotlib.pyplot as plt import datetime import scipy.misc from densenet_unet import _densenet_unet from read_data import _read_data from measurements import _measure densnet_unet_config=[6,8,8,8,6] compression_coefficient=.7 growth_rate=2 ext=''.join(map(str, densnet_unet_config)) _rd = _read_data() test_path='/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Data-01/prostate_test/' test_CTs, test_GTVs=_rd.read_imape_path(test_path) img_height=512 img_padded_size = 519 seg_size = 505 _meas=_measure() jj=[] dd=[] in_path='/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Code/dense_net_14/densenet_unet_output_volume_'+ext+'/' out_path='/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Code/dense_net_14/densenet_unet_output_volume_'+ext+'_Dilate/' # data_dir = [join(out_path, f) for f in listdir(out_path) if isfile(join(out_path, f))] labels=[] for img_indx in range(len(test_CTs)):
def dicejacc(): _rd = _read_data() test_path = '/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Data-01/prostate_test/' result_path='/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Code/dense_net_14/densenet_unet_output_volume_88888_0.5_2/' test_CTs, test_GTVs = _rd.read_imape_path(test_path) xtickNames = [] dice_boxplot=[] jacc_boxplot=[] for i in range(len(test_GTVs)): fig1, ax1 = plt.subplots() fig2, ax2 = plt.subplots() d = [] j = [] [CT_image, GTV_image, volume_depth, voxel_size, origin, direction] = _rd.read_image_seg_volume(test_CTs, test_GTVs, i) ss = str(test_CTs[i]).split("/") name = ss[8] + '_' + ss[9] xtickNames.append(name) res = sitk.ReadImage(result_path + name + '.mha') res = sitk.GetArrayFromImage(res) for jj in range(GTV_image.shape[0]): im1 = np.asarray(res[jj][:][:]).astype(np.bool) im2 = np.asarray(GTV_image[jj][:][:]).astype(np.bool) im2 = im2[0:511,0:511] im2 = im2[int(511/2)-int(im1.shape[0]/2)-1:int(511/2)+int(im1.shape[0]/2), int(511 / 2) - int(im1.shape[1]/2) - 1:int(511 / 2) + int(im1.shape[1]/2)] dice(im1, im2) d.append(dice(im1, im2)) j.append(jaccard(im1, im2)) index = np.arange(len(d)) bar_width = 0.35 opacity = 1 rects1 =ax1.bar(index, d, bar_width, alpha=1, color='b', label='Dice')#'bo--',range(2), j, 'rs--') rects2 =ax2.bar(index+bar_width, j, bar_width, alpha=1, color='r', label='Jaccard')#,range(2), j, 'rs--') # first_legend = plt.legend(handles=[line1], loc=1) # ax = plt.gca().add_artist(first_legend) ax1.set_xlabel('Slices') ax2.set_xlabel('Slices') ax1.set_ylabel('Accuracy') ax2.set_ylabel('Accuracy') non_zero_j = [x for x in j if x != 0] non_zero_d = [x for x in d if x != 0] if len(non_zero_j)!=0: title2=name+': '+'jaccard: (%.2f,%.2f)' %\ (min(non_zero_j), max(non_zero_j)) title1 = name + ': ' + 'dice: (%.2f,%.2f)' % \ ( min(non_zero_d), max(non_zero_d)) else: title2 = name + ': ' + 'jaccard: (%.2f,%.2f)' % \ (min(j), max(j)) title1 = name + ': ' + 'dice: (%.2f,%.2f)' % \ ( min(d), max(d)) fig1.suptitle(title1) ax1.legend( loc=4) plt.ylim(0, 1.1) fig1.savefig(result_path + '/dice_'+name+'.png') fig2.suptitle(title2) ax2.legend(loc=4) plt.ylim(0, 1.0) fig2.savefig(result_path + '/jaccard_'+name+'.png') dice_boxplot.append((non_zero_d)) jacc_boxplot.append((non_zero_j)) fig1.clear() fig2.clear() fig, ax = plt.subplots() plt.boxplot(dice_boxplot, 0, '') plt.xticks(list(range(1, len(dice_boxplot) + 1)), xtickNames, rotation='vertical') plt.ylabel('Dice') plt.margins(0.2) plt.subplots_adjust(bottom=.4) fig.savefig(result_path + '/dice_boxplot.png') fig, ax = plt.subplots() plt.boxplot(jacc_boxplot, 0, '') plt.xticks(list(range(1, len(jacc_boxplot) + 1)), xtickNames, rotation='vertical') plt.ylabel('Jaccard') plt.margins(0.2) plt.subplots_adjust(bottom=.4) fig.savefig(result_path + '/jacc_boxplot.png')
# img_name='CT_padded.mha' # label_name='GTV_CT_padded.mha' # torso_tag='CT_padded_Torso.mha' train_tag='train/' validation_tag='validation/' test_tag='Esophagus/' # img_name='CTpadded.mha' # label_name='GTV_CTpadded.mha' # torso_tag='Torsopadded.mha' img_name = '' label_name = '' torso_tag = '' _rd = _read_data(data=data,train_tag=train_tag, validation_tag=validation_tag, test_tag=test_tag, img_name=img_name, label_name=label_name) test_path = '/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Data-01/'+test_tag chckpnt_dir = '/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Code/'+Log+log_tag+'/densenet_unet_checkpoints/' # test_CTs, test_GTVs ,test_Torsos= _rd.read_imape_path(test_path) train_CTs, train_GTVs, train_Torso, train_penalize, \ validation_CTs, validation_GTVs, validation_Torso, validation_penalize, \ test_CTs, test_GTVs, test_Torso, test_penalize = _rd.read_data_path(fold=fold) # test_CTs=train_CTs # test_GTVs=train_GTVs # test_Torso=train_Torso # test_penalize=train_penalize
from joblib import Parallel, delayed import multiprocessing import pandas as pd def penalize_mask_generator(gtv_name, indx): nm = gtv_name.split('.mha')[0] + '.xlsx' print(indx, nm) [scan, voxel_size, origin, direction] = rd.read_volume(gtv_name) where = np.where(scan) df = pd.DataFrame({'x': where[0], 'y': where[1], 'z': where[2]}) writer = pd.ExcelWriter(nm) df.to_excel(writer, 'Sheet1') writer.save() rd = _read_data(2, train_tag='train/', validation_tag='validation/', test_tag='test/', img_name='CT_re113.mha', label_name='GTV_re113.mha') CTs, GTVs, Torsos = rd.read_imape_path2( '/srv/2-lkeb-17-dl01/syousefi/TestCode/EsophagusProject/Data-01/21data1100data2-v3/' ) num_cores = multiprocessing.cpu_count() res = Parallel(n_jobs=num_cores)(delayed(penalize_mask_generator)( gtv_name=GTVs[i], indx=i) for i in range(len(GTVs))) #len(GTVs)
def run_net(densnet_unet_config, compression_coefficient, growth_rate, img_padded_size, seg_size, Exp, log_ext, LOGDIR, train_acc_file, validation_acc_file, chckpnt_dir): '''read 2d images from the data:''' two_dim = True _rd = _read_data() flag = False # if os.path.exists(LOGDIR): # shutil.rmtree(LOGDIR) '''read path of the images for train, test, and validation''' train_CTs, train_GTVs, validation_CTs, validation_GTVs, \ test_CTs, test_GTVs=_rd.read_image_path() GTV_patchs_size = 81 patch_window = 95 sample_no = 100000 #badan avaz kon!!1000 batch_no = 50 batch_no_validation = 1000 validation_samples = 100000 # badan avaz kon!!1000 display_step = 100 run_validation_steps = display_step learning_rate = 1E-4 learning_decay = .95 if two_dim: image = tf.placeholder(tf.float32, shape=[None, None, None, 1]) label = tf.placeholder(tf.float32, shape=[None, None, None, 2]) # image=tf.placeholder(tf.float32,shape=[None,patch_window,patch_window,1]) # image=tf.placeholder(tf.float32,shape=[None,512,512,1]) # label=tf.placeholder(tf.float32,shape=[None,GTV_patchs_size,GTV_patchs_size,2]) # label=tf.placeholder(tf.float32,shape=[None,512,512,2]) ave_vali_acc = tf.placeholder(tf.float32) ave_loss_vali = tf.placeholder(tf.float32) dropout = tf.placeholder(tf.float32, name='dropout') is_training = tf.placeholder(tf.bool, name='is_training') dense_net_dim = tf.placeholder(tf.int32, name='dense_net_dim') # _u_net=_unet() # _u_net.unet(image) _dn = _densenet_unet(densnet_unet_config, compression_coefficient, growth_rate) #create object y = _dn.dens_net(image, is_training, dropout, dense_net_dim) sess = tf.Session() train_writer = tf.summary.FileWriter(LOGDIR + '/train' + Exp, graph=tf.get_default_graph()) validation_writer = tf.summary.FileWriter(LOGDIR + '/validation' + Exp, graph=sess.graph) # y=_dn.vgg(image) saver = tf.train.Saver() '''AdamOptimizer:''' with tf.name_scope('cost'): cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits( logits=y, labels=label), name="cost") tf.summary.scalar("cost", cost) # extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.name_scope('train'): optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost) with tf.name_scope('validation'): average_validation_accuracy = ave_vali_acc average_validation_loss = ave_loss_vali tf.summary.scalar("average_validation_accuracy", average_validation_accuracy) tf.summary.scalar("average_validation_loss", average_validation_loss) with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): correct_prediction = tf.equal(tf.argmax(y, 3), tf.argmax(label, 3)) with tf.name_scope('accuracy'): accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar("accuracy", accuracy) sess.run(tf.global_variables_initializer()) train_writer.add_graph(sess.graph) summ = tf.summary.merge_all() total_epochs = 10 img_width = 512 img_height = 512 # patch_radius = 49 '''loop for epochs''' itr1 = 0 for epoch in range(total_epochs): print('0') save_file(train_acc_file, 'epoch: %d\n' % (epoch)) save_file(validation_acc_file, 'epoch: %d\n' % (epoch)) print("epoch #: %d" % (epoch)) startTime = time.time() GTV_patchs, CT_image_patchs = _rd.read_data_all_train_batches( train_CTs, train_GTVs, sample_no, GTV_patchs_size, patch_window, img_width, img_height, epoch) validation_CT_image, validation_GTV_image = _rd.read_all_validation_batches( validation_CTs, validation_GTVs, validation_samples, GTV_patchs_size, patch_window, img_width, img_height, epoch + 1, img_padded_size, seg_size, whole_image=0) step = 0 '''loop for training batches''' while (step * batch_no < sample_no): # print('Training %d samples of %d' %(step*batch_no,sample_no)) train_CT_image_patchs = CT_image_patchs[step * batch_no:(step + 1) * batch_no - 1] train_GTV_label = GTV_patchs[step * batch_no:(step + 1) * batch_no - 1] [acc_train1, loss_train1, optimizing] = sess.run( [accuracy, cost, optimizer], feed_dict={ image: train_CT_image_patchs, label: train_GTV_label, dropout: 0.5, is_training: True, ave_vali_acc: -1, ave_loss_vali: -1, dense_net_dim: patch_window }) save_file(train_acc_file, '%f, %f\n' % (acc_train1, loss_train1)) if itr1 % display_step == 0: [sum_train] = sess.run( [summ], feed_dict={ image: train_CT_image_patchs, label: train_GTV_label, dropout: 0.5, is_training: True, ave_vali_acc: acc_train1, ave_loss_vali: loss_train1, dense_net_dim: patch_window }) train_writer.add_summary(sum_train, itr1) #=============validation================ '''Validation: ''' validation_step = 0 loss_validation = 0 acc_validation = 0 while (validation_step * batch_no_validation < validation_samples): validation_CT_image_patchs = validation_CT_image[ validation_step * batch_no_validation:(validation_step + 1) * batch_no_validation - 1] validation_GTV_label = validation_GTV_image[ validation_step * batch_no_validation:(validation_step + 1) * batch_no_validation - 1] [acc_vali, loss_vali] = sess.run( [accuracy, cost], feed_dict={ image: validation_CT_image_patchs, label: validation_GTV_label, dropout: 1, is_training: False, ave_vali_acc: -1, ave_loss_vali: -1, dense_net_dim: patch_window }) acc_validation += acc_vali loss_validation += loss_vali validation_step += 1 # print(validation_step * batch_no_validation ) save_file(validation_acc_file, '%f, %f\n' % (acc_vali, loss_vali)) # print('Validation, step: %d accuracy: %.4f loss: %f' % ( # itr1, acc_vali, loss_vali)) # end while acc_validation = acc_validation / validation_step loss_validation = loss_validation / validation_step print( '******Validation, step: %d accuracy: %.4f loss: %f*******' % (step, acc_validation, loss_validation)) [sum_validation] = sess.run( [summ], feed_dict={ image: validation_CT_image_patchs, label: validation_GTV_label, dropout: 1, is_training: False, ave_vali_acc: acc_validation, ave_loss_vali: loss_validation, dense_net_dim: patch_window }) validation_writer.add_summary(sum_validation, itr1) print('end of validation') #end if step = step + 1 itr1 = itr1 + 1 endTime = time.time() print("End of epoch----> %d, elapsed time: %d" % (epoch, endTime - startTime)) if itr1 % 1 == 0: if flag == False: img, seg = _rd.read_all_validation_batches(test_CTs, test_GTVs, validation_samples, GTV_patchs_size, patch_window, img_width, img_height, epoch + 1, img_padded_size, seg_size, whole_image=1) scipy.misc.imsave( './dense_unet_out/test.png', img[0][:][:].reshape(img_padded_size, img_padded_size)) scipy.misc.imsave('./dense_unet_out/label.png', (seg[0][:][:])[:, :, 1]) flag = True [acc_img, loss_img, out] = sess.run( [accuracy, cost, y], feed_dict={ image: img, label: seg, dropout: 1, is_training: False, ave_vali_acc: -1, ave_loss_vali: -1, dense_net_dim: img_padded_size }) # imgplot = plt.imshow((out[0][:][:])[:, :, 1], cmap='gray') # plt.figure() # img1=np.zeros((1, 1023, 1023)) # img1[0][:][:] = np.lib.pad((out[0][:][:])[:, :, 1], (263, 263), "constant", constant_values=(0, 0)) # imgplot = plt.imshow((out[0][:][:])[:, :, 1], cmap='gray') # plt.figure() # img1 = np.zeros((1, 1023, 1023)) # img1[0][:][:] = np.lib.pad((seg[0][:][:])[:, :, 1], (263, 263), "constant", constant_values=(0, 0)) # imgplot = plt.imshow((img1[0][:][:]), cmap='gray') # plt.figure() # imgplot = plt.imshow(img[0][:][:].reshape(1023, 1023), cmap='gray') scipy.misc.imsave('./dense_unet_out/result_%d.png' % (epoch), (out[0][:][:])[:, :, 1]) '''saveing model after each epoch''' chckpnt_path = os.path.join(chckpnt_dir, 'densenet_unet.ckpt') saver.save(sess, chckpnt_path, global_step=epoch) learning_rate = learning_rate * learning_decay