import nibabel as nib
import numpy as np
from scipy import ndimage
import os
from copy import deepcopy as dcopy
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter
from skimage.feature import local_binary_pattern as lbp
from ms_segmentation.general.general import save_image, list_folders, create_folder, list_files_with_name_containing
from ms_segmentation.data_generation.patch_manager_3d import normalize_data
from scipy.optimize import fmin
path_data = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\longitudinal'
path_histograms = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\chi_square_histograms'
path_write = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\chi_square_images'
create_folder(path_write)
create_folder(path_histograms)
patients = list_folders(path_data)
modalities = ["flair", "mprage", "pd", "t2"]
#tissues = ["wm", "gm", "mask1"]
tissues = ["brain_mask"]
to_analyze = {'preprocessed': 'nii'}
colors = ['b', 'g', 'r', 'c', 'm']
ref = []
wm_masks_ref = []
#f = lambda x: np.sum((np.histogram(fl_0[fl_0>0].ravel(), 256, [0,1], density=True)[0] - np.histogram(x[0]*fl_1[fl_1>0].ravel(), 256, [0,1], density=True)[0]**2)/(np.histogram(fl_0[fl_0>0].ravel(), 256, [0,1], density=True)[0] + 0.00001))
image_format = "nii.gz"
all_histograms = [[], [], [], []]
path_data = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS'
#path_data = r'D:\dev\INS1'
path_train = jp(path_data, "train")
path_test = jp(path_data, "test")
#path_test = r'D:\dev\ISBI_CS\test'
#path_test = r'D:\dev\INS_Test\test'
debug = True
if (debug):
experiment_name = "dummy_experiment_2DUNetConvGRU"
else:
experiment_name, curr_date, curr_time = get_experiment_name(
the_prefix="unet2dconvGRU")
#experiment_name = 'unet3d_2020-03-24_16_08_04[ISBI1-17]'
path_results = jp(path_data, experiment_name)
create_folder(path_results)
path_models = jp(path_results, "models")
create_folder(path_models)
path_segmentations = jp(path_results, "results")
create_folder(path_segmentations)
# Visualize one subject
#img, img_header = load(jp(path_train, '00001', 'FLAIR_masked.nii.gz'))
#mask, mask_header = load(jp(path_train, '00001', 'mask_FLAIR.nii.gz'))
#shim_overlay_slice(img, mask, 25, alpha=0.4)
#shim(img, 16)
# Define options for training
options = {}
# training data path
options['training_path'] = path_train
#path_test_cs = r'D:\dev\ms_data\Challenges\ISBI2015\sample1'
path_test_cs = r'D:\dev\ms_data\Preprocessed-AnonymPatData\cross_sectional' #ACHTUUUUNG
if path_test_cs.split("\\")[-1] == "histogram_matched":
print("Histogram matched test dataset is being used")
#path_test_l = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\longitudinal'
path_test_l = r'D:\dev\ms_data\Preprocessed-AnonymPatData\normalized_wm'
path_experiments_cs = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\cross_validation'
path_experiments_l = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\cross_validation'
#path_results = r'D:\dev\ms_data\Challenges\ISBI2015\res'
if experiment_type == 'cs':
path_results = r'D:\dev\ms_data\Preprocessed-AnonymPatData\results'
else:
path_results = r'D:\dev\ms_data\Preprocessed-AnonymPatData\results'
create_folder(path_results)
use_gpu = True
def get_result_name(the_paths, the_base):
accum = 0
for p in the_paths:
num_ensembles = len(list_folders(jp(p, "ensembles")))
num_non_ensembles = len(list_folders(p)) - 1 # minus ensembles folder
total = num_ensembles + num_non_ensembles
accum += total
new_name = the_base + str(accum+1).zfill(2)
return new_name
experiment_name_folder = get_result_name([r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\results_cs', r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\results_l'], "qwertz")
create_folder(jp(path_results, experiment_name_folder))
case_index = 1
for i_pat, pat in enumerate(patients):
print("Patient: ", i_pat+1, "/", len(patients))
path_pat = jp(path_origin, pat, "preprocessed")
flair_images = list_files_with_name_containing(path_pat, "flair", "nii")
mprage_images = list_files_with_name_containing(path_pat, "mprage", "nii")
pd_images = list_files_with_name_containing(path_pat, "pd", "nii")
t2_images = list_files_with_name_containing(path_pat, "t2", "nii")
#check that all modalities have the same number of images
assert len(flair_images) == len(mprage_images) == len(pd_images) == len(t2_images)
#create patient folder
create_folder(jp(path_cs, str(case_index).zfill(2)))
for i_img in range(len(flair_images)):
print("Timepoint: ", i_img+1)
# create timepoint folder
create_folder(jp(path_cs, str(case_index).zfill(2), str(i_img+1).zfill(2)))
shutil.copyfile(flair_images[i_img], jp(path_cs, str(case_index).zfill(2), str(i_img+1).zfill(2), "flair.nii"))
shutil.copyfile(mprage_images[i_img], jp(path_cs, str(case_index).zfill(2), str(i_img+1).zfill(2), "mprage.nii"))
shutil.copyfile(pd_images[i_img], jp(path_cs, str(case_index).zfill(2), str(i_img+1).zfill(2), "pd.nii"))
shutil.copyfile(t2_images[i_img], jp(path_cs, str(case_index).zfill(2), str(i_img+1).zfill(2), "t2.nii"))
case_index += 1
#change extension
for i_pat, pat in enumerate(patients):
import pandas as pd
import numpy as np
from copy import deepcopy as dc
import nibabel as nib
import matplotlib.pyplot as plt
import seaborn as sns
from ms_segmentation.general.general import create_folder, list_folders, list_files_with_name_containing, list_files_with_extension
from ms_segmentation.evaluation.metrics import compute_metrics
sns.set(style="whitegrid")
path_cs = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\cross_validation'
path_l = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\cross_validation'
path_gt = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\isbi_train'
path_results = r'D:\dev\ms_data\Challenges\ISBI2015\boxplots'
create_folder(path_results)
exp_cs = [
'CROSS_VALIDATION_UNet3D_2020-06-04_14_54_25[all_modalities_new]',
'CROSS_VALIDATION_UNet3D_2020-06-25_07_07_15[chi-square_norm_train]'
]
exp_l = [
'CROSS_VALIDATION_UNetConvLSTM3D_2020-06-23_21_31_39[longitudinal_chisquare_normalization_new]',
'CROSS_VALIDATION_UNetConvLSTM3D_2020-06-13_08_43_21[all_modalities_no_hist_matching_new]'
]
metrics = ["DSC", "LFPR", "LTPR"]
cases = {
"CS (min-max norm.)": jp(path_cs, exp_cs[0]),
"CS (proposed norm.)": jp(path_cs, exp_cs[0]),
"L (min-max norm.)": jp(path_l, exp_l[0]),
tmpl_values, tmpl_counts = np.unique(template.ravel(), return_counts=True)
# calculate normalized quantiles for each array
src_quantiles = np.cumsum(src_counts) / source.size
tmpl_quantiles = np.cumsum(tmpl_counts) / template.size
interp_a_values = np.interp(src_quantiles, tmpl_quantiles, tmpl_values)
return interp_a_values[src_unique_indices].reshape(source.shape)
patients = list_folders(path_data)
for pat in patients: # for each patient
print("Patient:", pat)
timepoints = list_folders(jp(path_data, pat))
create_folder(jp(path_new_data, pat))
#take first point as reference
ref_flair = normalize_data(
nib.load(jp(path_data, pat, timepoints[0],
'flair.nii.gz')).get_fdata())
ref_mprage = normalize_data(
nib.load(jp(path_data, pat, timepoints[0],
'mprage.nii.gz')).get_fdata())
ref_pd = normalize_data(
nib.load(jp(path_data, pat, timepoints[0], 'pd.nii.gz')).get_fdata())
ref_t2 = normalize_data(
nib.load(jp(path_data, pat, timepoints[0], 't2.nii.gz')).get_fdata())
brain_mask_ref = nib.load(
jp(path_data, pat, timepoints[0], 'brain_mask.nii.gz')).get_fdata()
#mask1 = nib.load(jp(path_data, pat, timepoints[0], 'mask1.nii.gz')).get_fdata()
#mask2 = nib.load(jp(path_data, pat, timepoints[0], 'mask2.nii.gz')).get_fdata()
# Ensemble of different models
import nibabel as nib
import numpy as np
import os
from os.path import join as jp
from ms_segmentation.general.general import save_image, list_folders, create_folder
path_base = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\cross_validation'
ensemble_name = 'aensemble2'
create_folder(jp(path_base, ensemble_name))
indexes_experiments = [13, 14, 15]
all_experiments = list_folders(path_base)
to_consider = [all_experiments[i] for i in indexes_experiments]
num_folds = 5
tp_amounts = []
data = dict().fromkeys(to_consider, None)
for k in data.keys():
data[k] = []
# read images:
for exp in to_consider: # for each of the selected experiments
for f in range(num_folds): # for each fold
timepoints = []
images = os.listdir(
jp(path_base, exp, "fold" + str(f + 1).zfill(2), "results",
str(f + 1).zfill(2))) # List all images for current fold
for img in images:
timepoints.append(
nib.load(
jp(path_base, exp, "fold" + str(f + 1).zfill(2), "results",
from os.path import join as jp
import pandas as pd
import numpy as np
import nibabel as nib
import SimpleITK as sitk
from ms_segmentation.general.general import list_folders, list_files_with_name_containing, create_folder
path_masks = r'D:\dev\s.tasconmorales\test_segm\PROCESS\INPUT\mri'
path_target = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\isbi_train'
path_copy = r'D:\dev\s.tasconmorales\wm_gm_masks'
patients = list_folders(path_masks)
for pat in patients:
print('Patient:', pat)
create_folder(jp(path_copy, pat))
n_timepoints = len(
list_files_with_name_containing(jp(path_target, pat), 'mprage',
'nii.gz'))
#read ref image
ref = sitk.ReadImage(jp(path_masks, pat, 'ref.nii.gz'))
#Read each mask
for tp in range(n_timepoints):
print('Time-point:', tp + 1)
gm = sitk.ReadImage(
jp(path_masks, pat, 'p1mprage_' + str(tp + 1).zfill(2) + '.nii'))
gm_np = sitk.GetArrayFromImage(gm)
gm_new = sitk.GetImageFromArray((gm_np > 0.5).astype(np.uint8))
gm_new.CopyInformation(ref)
sitk.WriteImage(
gm_new,
import os
import shutil
from ms_segmentation.general.general import list_folders, create_folder, cls
from os.path import join as jp
origin = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\histogram_matched'
destiny = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\histogram_matched'
to_copy = ['flair', 'pd', 'mprage', 't2', 'brain_mask', 'mask1', 'mask2']
patients = list_folders(origin)
for pat in patients:
create_folder(jp(destiny, pat))
#Copy all timepoints from origin to destiny
timepoints = list_folders(jp(origin, pat))
for tp in timepoints:
cls()
print("Patient: ", pat)
print("TP: ", tp)
for elem in to_copy:
shutil.copyfile(jp(origin, pat, tp, elem + '.nii.gz'),
jp(destiny, pat, elem + '_' + tp + '.nii.gz'))
import os
import numpy as np
import nibabel as nib
from os.path import join as jp
from ms_segmentation.general.general import save_image, list_folders, create_log, create_folder, list_files_with_extension
path_base = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\results_cs'
path_ensembles = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\results_cs\ensembles'
experiments_to_ensemble = ['qwertz06', 'qwertz09', 'qwertz11']
num_ens = len(list_folders(path_ensembles))
name_folder_new_ensemble = "Ensemble_" + str(num_ens + 1).zfill(2)
create_folder(jp(path_ensembles, name_folder_new_ensemble))
# Check that listed experiments exist
all_experiments = list_folders(path_base)
all_experiments = [x for x in all_experiments if x in experiments_to_ensemble]
assert len(all_experiments) % 2 != 0 # number of experiments must be odd
all_images = []
for i_exp, exp in enumerate(all_experiments):
print("Current experiment: ", exp, i_exp + 1, "/", len(all_experiments))
all_images_names = list_files_with_extension(jp(path_base, exp), "nii")
all_images_curr_exp = []
for img in all_images_names:
all_images_curr_exp.append(
nib.load(jp(path_base, exp, img)).get_fdata().astype(np.uint8))
all_images.append(np.stack(all_images_curr_exp)) # (61, volume size)
all_images_np = np.stack(all_images)
import os
from os.path import join as jp
import shutil
from ms_segmentation.general.general import list_folders, create_folder
path_origin = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\chi_square_images_longit'
path_destiny = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\chi_square_images_cs'
create_folder(path_destiny)
patients = list_folders(path_origin)
def get_name(the_string):
if "flair" in the_string:
return "flair_norm.nii.gz"
elif "brain_mask" in the_string:
return "brain_mask.nii.gz"
elif "pd" in the_string:
return "pd_norm.nii.gz"
elif "mprage" in the_string:
return "mprage_norm.nii.gz"
elif "t2" in the_string:
return "t2_norm.nii.gz"
elif "mask1" in the_string:
return "mask1.nii.gz"
elif "mask2" in the_string:
return "mask2.nii.gz"
else:
raise Exception("Unknown image type")
for pat in patients:
create_folder(jp(path_destiny, pat))
all_files = os.listdir(jp(path_origin, pat))
