def get_result_name(the_paths, the_base): accum = 0 for p in the_paths: num_ensembles = len(list_folders(jp(p, "ensembles"))) num_non_ensembles = len(list_folders(p)) - 1 # minus ensembles folder total = num_ensembles + num_non_ensembles accum += total new_name = the_base + str(accum+1).zfill(2) return new_name
import pandas as pd import numpy as np import nibabel as nib import cc3d from ms_segmentation.general.general import list_folders, list_files_with_name_containing from ms_segmentation.evaluation.metrics import compute_metrics last_only = True exp_folders = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\cross_validation' gt_folder = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\isbi_train' gt_list = ['mask1', 'mask2'] if last_only: all_experiments = list_folders(exp_folders)[-1:] else: all_experiments = list_folders(exp_folders) for i_exp, exp in enumerate(all_experiments): print("Curr. exp: ", exp, "(", i_exp + 1, "/", len(all_experiments), ")") for gt_curr in gt_list: experiment_folder = jp(exp_folders, exp) #gt_folder = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\isbi_train' gt_name = gt_curr patients = list_folders(gt_folder) post_processing = True post_processing_type = 'remove_small' min_area = 3 folds = list_folders(experiment_folder) labels_for_df = compute_metrics(None, None, labels_only=True)
import os import cv2 as cv import numpy as np import nibabel as nib from os.path import join as jp from ms_segmentation.general.general import save_image, list_folders path_base = r'D:\dev\s.tasconmorales\feature_maps' experiments = list_folders(path_base) batch_size = 32 patch_slice = 16 all_fm = {} for exp in experiments: all_fm[exp] = [] for i in range(batch_size): all_fm[exp].append( nib.load( jp(path_base, exp, "inc_batch000_fm" + str(i).zfill(3) + ".nii.gz")).get_fdata()[:, :, patch_slice]) for k, v in all_fm.items(): all_fm[k] = np.concatenate(v, axis=1) tot = np.concatenate(list(all_fm.values()), axis=0) def bring_to_256_levels(input_im): return (255 * (input_im - np.min(input_im)) / (np.max(input_im) - np.min(input_im))).astype(np.uint8)
import numpy as np import nibabel as nib import os import shutil from os.path import join as jp from ms_segmentation.general.general import list_folders, list_files_with_name_containing, create_folder path_base = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images' path_cs = jp(path_base, 'cross_sectional') path_origin = jp(path_base, 'orig') patients = list_folders(path_origin) case_index = 1 for i_pat, pat in enumerate(patients): print("Patient: ", i_pat+1, "/", len(patients)) path_pat = jp(path_origin, pat, "preprocessed") flair_images = list_files_with_name_containing(path_pat, "flair", "nii") mprage_images = list_files_with_name_containing(path_pat, "mprage", "nii") pd_images = list_files_with_name_containing(path_pat, "pd", "nii") t2_images = list_files_with_name_containing(path_pat, "t2", "nii") #check that all modalities have the same number of images assert len(flair_images) == len(mprage_images) == len(pd_images) == len(t2_images) #create patient folder create_folder(jp(path_cs, str(case_index).zfill(2))) for i_img in range(len(flair_images)): print("Timepoint: ", i_img+1) # create timepoint folder
import os from os.path import join as jp import numpy as np import SimpleITK as sitk from ms_segmentation.general.general import list_folders path_data = r'D:\dev\ms_data\Preprocessed-AnonymPatData' cases_list = list_folders(path_data) for case in cases_list: print("Current case: ", case) tps = list_folders(jp(path_data, case)) for tp in tps: files_list = os.listdir(jp(path_data, case, tp)) for file_ in files_list: if "gz" in file_: continue # Read image img_itk = sitk.ReadImage(jp(path_data, case, tp, file_)) # Save with new format sitk.WriteImage(img_itk, jp(path_data, case, tp, file_ + ".gz")) # Build brain image from FLAIR image (simple thresholding) if "flair" in file_ and "brain_mask.nii.gz" not in files_list: flair_np = sitk.GetArrayFromImage(img_itk) mask = flair_np > 0 mask_itk = sitk.GetImageFromArray(mask.astype(np.uint8)) mask_itk.CopyInformation(img_itk) sitk.WriteImage(mask_itk, jp(path_data, case, tp, "brain_mask.nii.gz")) os.remove(jp(path_data, case, tp, file_))
import os import nibabel as nib import numpy as np from os.path import join as jp from ms_segmentation.general.general import save_image, list_folders, list_files_with_name_containing from ms_segmentation.data_generation.patch_manager_3d import normalize_data path_base = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\cross_sectional' cross_sectional = True patients = list_folders(path_base) if cross_sectional: for pat in patients: print("Patient ", pat) tps = list_folders(jp(path_base, pat)) for tp in tps: print("TP: ", tp) flair = normalize_data( nib.load(jp(path_base, pat, tp, 'flair.nii.gz')).get_fdata()) t2 = normalize_data( nib.load(jp(path_base, pat, tp, 't2.nii.gz')).get_fdata()) pd = normalize_data( nib.load(jp(path_base, pat, tp, 'pd.nii.gz')).get_fdata()) t1 = normalize_data( nib.load(jp(path_base, pat, tp, 'mprage.nii.gz')).get_fdata()) t1_inv = normalize_data(np.max(t1) - t1) t2_flair = normalize_data(flair * t2) pd_flair = normalize_data(flair * pd) t1_inv_flair = normalize_data(flair * t1_inv) what = normalize_data(flair * (t2 + pd + t1_inv))
import os from os.path import join as jp import pandas as pd import numpy as np import nibabel as nib import cc3d from ms_segmentation.general.general import list_folders, list_files_with_name_containing from ms_segmentation.evaluation.metrics import compute_metrics last_only = True # evaluate only last experiment exp_folders = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\cross_validation' gt_list = ['mask1.nii.gz', 'mask2.nii.gz'] all_experiments = list_folders(exp_folders) if last_only: all_experiments = all_experiments[-1:] for i_exp, exp in enumerate(all_experiments): print("Curr. exp: ", exp, "(", i_exp + 1, "/", len(all_experiments), ")") for gt_curr in gt_list: experiment_folder = jp(exp_folders, exp) gt_folder = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\isbi_cs' gt_name = gt_curr patients = list_folders(gt_folder) post_processing = True post_processing_type = 'remove_small' min_area = 3 folds = list_folders(experiment_folder) labels_for_df = compute_metrics(None, None, labels_only=True) global_df = pd.DataFrame(columns=labels_for_df) cnt_global = 0
import os import shutil from ms_segmentation.general.general import list_folders, create_folder, cls from os.path import join as jp origin = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS\histogram_matched' destiny = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\histogram_matched' to_copy = ['flair', 'pd', 'mprage', 't2', 'brain_mask', 'mask1', 'mask2'] patients = list_folders(origin) for pat in patients: create_folder(jp(destiny, pat)) #Copy all timepoints from origin to destiny timepoints = list_folders(jp(origin, pat)) for tp in timepoints: cls() print("Patient: ", pat) print("TP: ", tp) for elem in to_copy: shutil.copyfile(jp(origin, pat, tp, elem + '.nii.gz'), jp(destiny, pat, elem + '_' + tp + '.nii.gz'))
if(num_timepoints%2==0): #even indexes.append(i+1) else: indexes.append(i) sorted_indexes = np.argsort(indexes) sorted_list_inference_patches = [list_inference_patches[k] for k in sorted_indexes] return sorted_list_inference_patches # Process experiment if experiment_type == 'cs': # if experiment is cross-sectional # cross-sectional path_exp = jp(path_experiments_cs, experiment_name) folds = list_folders(path_exp) #Read log file to get parameters parameters_dict = parse_log_file(jp(path_exp, folds[0])) # take file of first fold as reference all_folds = [] for f in folds: if selection[f] == False: continue fold_segmentations = [] # create model lesion_model = eval(parameters_dict["model_name"])(n_channels=len(eval(parameters_dict['input_data'])), n_classes=2, bilinear = False) lesion_model.cuda() # try to load the weights
options['norm_type'] = 'zero_one' options['batch_size'] = 16 options['patience'] = 20 #Patience for the early stopping options['gpu_use'] = True options['num_epochs'] = 200 options['optimizer'] = 'adam' options[ 'patch_sampling'] = 'mask' # (mask, balanced or balanced+roi or non-uniform) options['loss'] = 'dice' # (dice, cross-entropy, categorical-cross-entropy) options['resample_each_epoch'] = False path_base = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_CS' path_data = jp(path_base, 'isbi_cs') options['path_data'] = path_data path_res = jp(path_base, "cross_validation") all_patients = list_folders(path_data) if (debug): experiment_name = "dummy_experiment_CV_3DUNet" else: experiment_name, curr_date, curr_time = get_experiment_name( the_prefix="CROSS_VALIDATION_UNet3D") # Visualize feature maps activation = {} def get_activation(name): def hook(model, input, output): activation[name] = output.detach()
## Script to rename product images that contain "flair" in their names so that when listing files they are not liste # twice. import os from os.path import join as jp from ms_segmentation.general.general import list_folders, list_files_with_name_containing path_images = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\isbi_train' patients = list_folders(path_images) #changes = {'pd_times_flair': 'pd_times_fl', 't2_times_flair': 't2_times_fl', 't1_inv_times_flair': 't1_inv_times_fl', 'sum_times_flair': 'sum_times_fl'} changes ={'pd_times_fl': 'p_times_fl','t2_times_fl': 't_two_times_fl'} for pat in patients: # for each patient print("patient: ", pat) for k,v in changes.items(): list_of_files = list_files_with_name_containing(jp(path_images, pat), k, "nii.gz") for i_tp in range(len(list_of_files)): # for each timepoint print("timepoint: ", i_tp+1) new_name = list_of_files[i_tp].replace(k, v) os.rename(list_of_files[i_tp], new_name)
its values matches the cumulative density function of the template. """ src_values, src_unique_indices, src_counts = np.unique(source.ravel(), return_inverse=True, return_counts=True) tmpl_values, tmpl_counts = np.unique(template.ravel(), return_counts=True) # calculate normalized quantiles for each array src_quantiles = np.cumsum(src_counts) / source.size tmpl_quantiles = np.cumsum(tmpl_counts) / template.size interp_a_values = np.interp(src_quantiles, tmpl_quantiles, tmpl_values) return interp_a_values[src_unique_indices].reshape(source.shape) patients = list_folders(path_data) for pat in patients: # for each patient print("Patient:", pat) timepoints = list_folders(jp(path_data, pat)) create_folder(jp(path_new_data, pat)) #take first point as reference ref_flair = normalize_data( nib.load(jp(path_data, pat, timepoints[0], 'flair.nii.gz')).get_fdata()) ref_mprage = normalize_data( nib.load(jp(path_data, pat, timepoints[0], 'mprage.nii.gz')).get_fdata()) ref_pd = normalize_data( nib.load(jp(path_data, pat, timepoints[0], 'pd.nii.gz')).get_fdata()) ref_t2 = normalize_data(
import os import SimpleITK as sitk from os.path import join as jp import pandas as pd from ms_segmentation.general import general as misc path_orig = r'D:\dev\ms_data\Inspiration\INSPIRATION-org' path_proc = r'D:\dev\ms_data\Inspiration\INSPIRATION' # first, process original images centers = misc.list_folders(path_orig) columns = ['Center', 'Patient', 'Time Point', 'Modality', 'Size'] df_orig = pd.DataFrame(columns=columns) i_orig = 0 for i_c, c in enumerate(centers): # for each study center patients = misc.list_folders(jp(path_orig, c)) # list of patients for center c for i_pat, pat in enumerate(patients): #for every patient timepoints = misc.list_folders(jp(path_orig, c, pat)) #list of timepoints for tp in timepoints: modalities = misc.list_folders(jp( path_orig, c, pat, tp)) # List of available modalities for mod in modalities: imgs = misc.list_files_with_extension( jp(path_orig, c, pat, tp, mod), "nii.gz") # List nii.gz images if len(imgs ) > 1: # If more than one image for a certain modality df_orig.loc[i_orig] = [c, pat, tp, mod, 'multiple'] i_orig += 1
#plot_learning_curve(train_accs, val_accs, the_title="Accuracy plot", measure = "Accuracy", early_stopping = False, filename = jp(path_results, "acc_plot.png")) plot_learning_curve(train_jaccs, val_jaccs, the_title="Jaccard plot", measure = "Jaccard", early_stopping = False, filename = jp(path_results, "jaccard_plot.png")) else: try: lesion_model.load_state_dict(torch.load(jp(path_models,"checkpoint.pt"))) except: raise ValueError("No model found") #Evaluate all test images columns = columns = compute_metrics(None, None, labels_only=True) df = pd.DataFrame(columns = columns) test_images = list_folders(path_test) i_row=0 cnt=0 for case in test_images: print(cnt+1, "/", len(test_images)) path_timepoints = jp(path_test, case) list_images = get_dictionary_with_paths_cs([case], path_test, options['input_data']) list_rois = get_dictionary_with_paths_cs([case], path_test, [options['brain_mask']]) patch_half = tuple([idx // 2 for idx in options['patch_size']]) timepoints = list_folders(path_timepoints) # get candidate voxels all_infer_patches, all_coordenates = get_inference_patches( path_test=path_test,
import os from os.path import join as jp import pandas as pd import numpy as np import nibabel as nib import SimpleITK as sitk from ms_segmentation.general.general import list_folders, list_files_with_name_containing, create_folder path_masks = r'D:\dev\s.tasconmorales\test_segm\PROCESS\INPUT\mri' path_target = r'D:\dev\ms_data\Challenges\ISBI2015\ISBI_L\isbi_train' path_copy = r'D:\dev\s.tasconmorales\wm_gm_masks' patients = list_folders(path_masks) for pat in patients: print('Patient:', pat) create_folder(jp(path_copy, pat)) n_timepoints = len( list_files_with_name_containing(jp(path_target, pat), 'mprage', 'nii.gz')) #read ref image ref = sitk.ReadImage(jp(path_masks, pat, 'ref.nii.gz')) #Read each mask for tp in range(n_timepoints): print('Time-point:', tp + 1) gm = sitk.ReadImage( jp(path_masks, pat, 'p1mprage_' + str(tp + 1).zfill(2) + '.nii')) gm_np = sitk.GetArrayFromImage(gm) gm_new = sitk.GetImageFromArray((gm_np > 0.5).astype(np.uint8)) gm_new.CopyInformation(ref) sitk.WriteImage(
import os from copy import deepcopy as dcopy import matplotlib.pyplot as plt from scipy.ndimage.filters import gaussian_filter from skimage.feature import local_binary_pattern as lbp from ms_segmentation.general.general import save_image, list_folders, create_folder, list_files_with_name_containing from ms_segmentation.data_generation.patch_manager_3d import normalize_data from scipy.optimize import fmin path_data = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\longitudinal' path_histograms = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\chi_square_histograms' path_write = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\chi_square_images' create_folder(path_write) create_folder(path_histograms) patients = list_folders(path_data) modalities = ["flair", "mprage", "pd", "t2"] #tissues = ["wm", "gm", "mask1"] tissues = ["brain_mask"] to_analyze = {'preprocessed': 'nii'} colors = ['b', 'g', 'r', 'c', 'm'] ref = [] wm_masks_ref = [] #f = lambda x: np.sum((np.histogram(fl_0[fl_0>0].ravel(), 256, [0,1], density=True)[0] - np.histogram(x[0]*fl_1[fl_1>0].ravel(), 256, [0,1], density=True)[0]**2)/(np.histogram(fl_0[fl_0>0].ravel(), 256, [0,1], density=True)[0] + 0.00001)) image_format = "nii.gz" all_histograms = [[], [], [], []] all_histograms_aligned = [[], [], [], []] all_histograms_wm = [[], [], [], []] all_histograms_gm = [[], [], [], []]
the_title="Jaccard plot", measure="Jaccard", early_stopping=False, filename=jp(path_results, "jaccard_plot.png")) else: try: lesion_model.load_state_dict( torch.load(jp(path_models, "checkpoint.pt"))) except: raise ValueError("No model found") #Evaluate all images columns = ['Case', 'DSC', 'HD'] df = pd.DataFrame(columns=columns) test_images = list_folders(path_test) i_row = 0 cnt = 0 for case in test_images: print(cnt + 1, "/", len(test_images)) scan_path = jp(path_test, case) inf_slices = get_inference_slices(scan_path=scan_path, input_data=options['input_data'], normalize=True, norm_type='zero_one') probs = get_probs(inf_slices, lesion_model, device, options['num_classes'], options) labels = np.transpose(np.argmax(probs, axis=1).astype(np.uint8), (1, 2, 0)) labels_gt = nib.load(jp(path_test, case,
import os import numpy as np import nibabel as nib from os.path import join as jp from ms_segmentation.general.general import save_image, list_folders, create_log, create_folder, list_files_with_extension path_base = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\results_cs' path_ensembles = r'D:\dev\ms_data\Challenges\ISBI2015\Test_Images\results_cs\ensembles' experiments_to_ensemble = ['qwertz06', 'qwertz09', 'qwertz11'] num_ens = len(list_folders(path_ensembles)) name_folder_new_ensemble = "Ensemble_" + str(num_ens + 1).zfill(2) create_folder(jp(path_ensembles, name_folder_new_ensemble)) # Check that listed experiments exist all_experiments = list_folders(path_base) all_experiments = [x for x in all_experiments if x in experiments_to_ensemble] assert len(all_experiments) % 2 != 0 # number of experiments must be odd all_images = [] for i_exp, exp in enumerate(all_experiments): print("Current experiment: ", exp, i_exp + 1, "/", len(all_experiments)) all_images_names = list_files_with_extension(jp(path_base, exp), "nii") all_images_curr_exp = [] for img in all_images_names: all_images_curr_exp.append( nib.load(jp(path_base, exp, img)).get_fdata().astype(np.uint8)) all_images.append(np.stack(all_images_curr_exp)) # (61, volume size) all_images_np = np.stack(all_images)