def save_multilabel_img_as_multiple_files_endings(HP, img, affine, path): bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:] for idx, bundle in enumerate(bundles): img_seg = nib.Nifti1Image(img[:, :, :, idx], affine) ExpUtils.make_dir(join(path, "endings_segmentations")) nib.save(img_seg, join(path, "endings_segmentations", bundle + ".nii.gz"))
def save_fusion_nifti_as_npy(): #Can leave this always the same (for 270g and 32g) class HP: DATASET = "HCP" RESOLUTION = "1.25mm" FEATURES_FILENAME = "270g_125mm_peaks" LABELS_TYPE = np.int16 LABELS_FILENAME = "bundle_masks" DATASET_FOLDER = "HCP" #change this for 270g and 32g DIFFUSION_FOLDER = "32g_25mm" subjects = get_all_subjects() # fold0 = ['687163', '685058', '683256', '680957', '679568', '677968', '673455', '672756', '665254', '654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469'] # fold1 = ['992774', '991267', '987983', '984472', '983773', '979984', '978578', '965771', '965367', '959574', '958976', '957974', '951457', '932554', '930449', '922854', '917255', '912447', '910241', '907656', '904044'] # fold2 = ['901442', '901139', '901038', '899885', '898176', '896879', '896778', '894673', '889579', '887373', '877269', '877168', '872764', '872158', '871964', '871762', '865363', '861456', '859671', '857263', '856766'] # fold3 = ['849971', '845458', '837964', '837560', '833249', '833148', '826454', '826353', '816653', '814649', '802844', '792766', '792564', '789373', '786569', '784565', '782561', '779370', '771354', '770352', '765056'] # fold4 = ['761957', '759869', '756055', '753251', '751348', '749361', '748662', '748258', '742549', '734045', '732243', '729557', '729254', '715647', '715041', '709551', '705341', '704238', '702133', '695768', '690152'] # subjects = fold2 + fold3 + fold4 # subjects = ['654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469'] print("\n\nProcessing Data...") for s in subjects: print("processing data subject {}".format(s)) start_time = time.time() data = nib.load( join(C.NETWORK_DRIVE, "HCP_fusion_" + DIFFUSION_FOLDER, s + "_probmap.nii.gz")).get_data() print("Done Loading") data = np.nan_to_num(data) data = DatasetUtils.scale_input_to_unet_shape( data, HP.DATASET, HP.RESOLUTION) data = data[:-1, :, : -1, :] # cut one pixel at the end, because in scale_input_to_world_shape we ouputted 146 -> one too much at the end ExpUtils.make_dir( join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s)) np.save( join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s, DIFFUSION_FOLDER + "_xyz.npy"), data) print("Took {}s".format(time.time() - start_time)) print("processing seg subject {}".format(s)) start_time = time.time() # seg = ImgUtils.create_multilabel_mask(HP, s, labels_type=HP.LABELS_TYPE) seg = nib.load( join(C.NETWORK_DRIVE, "HCP_for_training_COPY", s, HP.LABELS_FILENAME + ".nii.gz")).get_data() if HP.RESOLUTION == "2.5mm": seg = ImgUtils.resize_first_three_dims(seg, order=0, zoom=0.5) seg = DatasetUtils.scale_input_to_unet_shape( seg, HP.DATASET, HP.RESOLUTION) np.save( join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s, "bundle_masks.npy"), seg) print("Took {}s".format(time.time() - start_time))
def precompute_batches(custom_type=None): ''' 9000 slices per epoch -> 200 batches (batchsize=44) per epoch => 200-1000 batches needed 270g_125mm_bundle_peaks_Y: no DAug, no Norm, only Y All_sizes_DAug_XYZ: 12g, 90g, 270g, DAug (no rotation, no elastic deform), Norm, XYZ 270g_125mm_bundle_peaks_XYZ: no DAug, Norm, XYZ ''' class HP: NORMALIZE_DATA = True DATA_AUGMENTATION = False CV_FOLD = 0 INPUT_DIM = (144, 144) BATCH_SIZE = 44 DATASET_FOLDER = "HCP" TYPE = "single_direction" EXP_PATH = "~" LABELS_FILENAME = "bundle_peaks" FEATURES_FILENAME = "270g_125mm_peaks" DATASET = "HCP" RESOLUTION = "1.25mm" LABELS_TYPE = np.float32 HP.TRAIN_SUBJECTS, HP.VALIDATE_SUBJECTS, HP.TEST_SUBJECTS = ExpUtils.get_cv_fold(HP.CV_FOLD) num_batches_base = 5000 num_batches = { "train": num_batches_base, "validate": int(num_batches_base / 3.), "test": int(num_batches_base / 3.), } if custom_type is None: types = ["train", "validate", "test"] else: types = [custom_type] for type in types: dataManager = DataManagerTrainingNiftiImgs(HP) batch_gen = dataManager.get_batches(batch_size=HP.BATCH_SIZE, type=type, subjects=getattr(HP, type.upper() + "_SUBJECTS"), num_batches=num_batches[type]) for idx, batch in enumerate(batch_gen): print("Processing: {}".format(idx)) # DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_Y" # DATASET_DIR = "HCP_batches/All_sizes_DAug_XYZ" DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_XYZ" ExpUtils.make_dir(join(C.HOME, DATASET_DIR, type)) data = nib.Nifti1Image(batch["data"], ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION)) nib.save(data, join(C.HOME, DATASET_DIR, type, "batch_" + str(idx) + "_data.nii.gz")) seg = nib.Nifti1Image(batch["seg"], ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION)) nib.save(seg, join(C.HOME, DATASET_DIR, type, "batch_" + str(idx) + "_seg.nii.gz"))
def save_multilabel_img_as_multiple_files_peaks(HP, img, affine, path): bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:] for idx, bundle in enumerate(bundles): data = img[:, :, :, (idx * 3):(idx * 3) + 3] if HP.FLIP_OUTPUT_PEAKS: data[:, :, :, 2] *= -1 # flip z Axis for correct view in MITK filename = bundle + "_f.nii.gz" else: filename = bundle + ".nii.gz" img_seg = nib.Nifti1Image(data, affine) ExpUtils.make_dir(join(path, "TOM")) nib.save(img_seg, join(path, "TOM", filename))
def save_multilabel_img_as_multiple_files_peaks(HP, img, affine, path): bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:] for idx, bundle in enumerate(bundles): data = img[:, :, :, (idx*3):(idx*3)+3] if HP.FLIP_OUTPUT_PEAKS: data[:, :, :, 2] *= -1 # flip z Axis for correct view in MITK filename = bundle + "_f.nii.gz" else: filename = bundle + ".nii.gz" img_seg = nib.Nifti1Image(data, affine) ExpUtils.make_dir(join(path, "TOM")) nib.save(img_seg, join(path, "TOM", filename))
def save_fusion_nifti_as_npy(): #Can leave this always the same (for 270g and 32g) class HP: DATASET = "HCP" RESOLUTION = "1.25mm" FEATURES_FILENAME = "270g_125mm_peaks" LABELS_TYPE = np.int16 LABELS_FILENAME = "bundle_masks" DATASET_FOLDER = "HCP" #change this for 270g and 32g DIFFUSION_FOLDER = "32g_25mm" subjects = get_all_subjects() # fold0 = ['687163', '685058', '683256', '680957', '679568', '677968', '673455', '672756', '665254', '654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469'] # fold1 = ['992774', '991267', '987983', '984472', '983773', '979984', '978578', '965771', '965367', '959574', '958976', '957974', '951457', '932554', '930449', '922854', '917255', '912447', '910241', '907656', '904044'] # fold2 = ['901442', '901139', '901038', '899885', '898176', '896879', '896778', '894673', '889579', '887373', '877269', '877168', '872764', '872158', '871964', '871762', '865363', '861456', '859671', '857263', '856766'] # fold3 = ['849971', '845458', '837964', '837560', '833249', '833148', '826454', '826353', '816653', '814649', '802844', '792766', '792564', '789373', '786569', '784565', '782561', '779370', '771354', '770352', '765056'] # fold4 = ['761957', '759869', '756055', '753251', '751348', '749361', '748662', '748258', '742549', '734045', '732243', '729557', '729254', '715647', '715041', '709551', '705341', '704238', '702133', '695768', '690152'] # subjects = fold2 + fold3 + fold4 # subjects = ['654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469'] print("\n\nProcessing Data...") for s in subjects: print("processing data subject {}".format(s)) start_time = time.time() data = nib.load(join(C.NETWORK_DRIVE, "HCP_fusion_" + DIFFUSION_FOLDER, s + "_probmap.nii.gz")).get_data() print("Done Loading") data = np.nan_to_num(data) data = DatasetUtils.scale_input_to_unet_shape(data, HP.DATASET, HP.RESOLUTION) data = data[:-1, :, :-1, :] # cut one pixel at the end, because in scale_input_to_world_shape we ouputted 146 -> one too much at the end ExpUtils.make_dir(join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s)) np.save(join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s, DIFFUSION_FOLDER + "_xyz.npy"), data) print("Took {}s".format(time.time() - start_time)) print("processing seg subject {}".format(s)) start_time = time.time() # seg = ImgUtils.create_multilabel_mask(HP, s, labels_type=HP.LABELS_TYPE) seg = nib.load(join(C.NETWORK_DRIVE, "HCP_for_training_COPY", s, HP.LABELS_FILENAME + ".nii.gz")).get_data() if HP.RESOLUTION == "2.5mm": seg = ImgUtils.resize_first_three_dims(seg, order=0, zoom=0.5) seg = DatasetUtils.scale_input_to_unet_shape(seg, HP.DATASET, HP.RESOLUTION) np.save(join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s, "bundle_masks.npy"), seg) print("Took {}s".format(time.time() - start_time))
def copy_training_files_to_ssd(HP, data_path): def id_generator(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) target_data_path = join("/ssd/", "tmp_" + id_generator(), HP.DATASET_FOLDER) ExpUtils.make_dir(join(target_data_path)) #get all folders in data_path directory subjects = [os.path.basename(os.path.normpath(d)) for d in glob(data_path + "/*/")] for subject in subjects: src = join(data_path, subject, HP.FEATURES_FILENAME) target = join(target_data_path, subject, HP.FEATURES_FILENAME) print("cp: {} -> {}".format(src, target)) # shutil.copyfile(src, target) return target_data_path
def copy_training_files_to_ssd(HP, data_path): def id_generator(size=6, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) target_data_path = join("/ssd/", "tmp_" + id_generator(), HP.DATASET_FOLDER) ExpUtils.make_dir(join(target_data_path)) #get all folders in data_path directory subjects = [ os.path.basename(os.path.normpath(d)) for d in glob(data_path + "/*/") ] for subject in subjects: src = join(data_path, subject, HP.FEATURES_FILENAME) target = join(target_data_path, subject, HP.FEATURES_FILENAME) print("cp: {} -> {}".format(src, target)) # shutil.copyfile(src, target) return target_data_path
def save_multilabel_img_as_multiple_files_endings_OLD(HP, img, affine, path, multilabel=True): ''' multilabel True: save as 1 and 2 without fourth dimension multilabel False: save with beginnings and endings combined ''' # bundles = ExpUtils.get_bundle_names("20")[1:] bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:] for idx, bundle in enumerate(bundles): data = img[:, :, :, (idx * 2):(idx * 2) + 2] > 0 multilabel_img = np.zeros(data.shape[:3]) if multilabel: multilabel_img[data[:, :, :, 0]] = 1 multilabel_img[data[:, :, :, 1]] = 2 else: multilabel_img[data[:, :, :, 0]] = 1 multilabel_img[data[:, :, :, 1]] = 1 img_seg = nib.Nifti1Image(multilabel_img, affine) ExpUtils.make_dir(join(path, "endings")) nib.save(img_seg, join(path, "endings", bundle + ".nii.gz"))
def save_multilabel_img_as_multiple_files_endings_OLD(HP, img, affine, path, multilabel=True): ''' multilabel True: save as 1 and 2 without fourth dimension multilabel False: save with beginnings and endings combined ''' # bundles = ExpUtils.get_bundle_names("20")[1:] bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:] for idx, bundle in enumerate(bundles): data = img[:, :, :, (idx * 2):(idx * 2) + 2] > 0 multilabel_img = np.zeros(data.shape[:3]) if multilabel: multilabel_img[data[:, :, :, 0]] = 1 multilabel_img[data[:, :, :, 1]] = 2 else: multilabel_img[data[:, :, :, 0]] = 1 multilabel_img[data[:, :, :, 1]] = 1 img_seg = nib.Nifti1Image(multilabel_img, affine) ExpUtils.make_dir(join(path, "endings")) nib.save(img_seg, join(path, "endings", bundle + ".nii.gz"))
def precompute_batches(custom_type=None): ''' 9000 slices per epoch -> 200 batches (batchsize=44) per epoch => 200-1000 batches needed 270g_125mm_bundle_peaks_Y: no DAug, no Norm, only Y All_sizes_DAug_XYZ: 12g, 90g, 270g, DAug (no rotation, no elastic deform), Norm, XYZ 270g_125mm_bundle_peaks_XYZ: no DAug, Norm, XYZ ''' class HP: NORMALIZE_DATA = True DATA_AUGMENTATION = False CV_FOLD = 0 INPUT_DIM = (144, 144) BATCH_SIZE = 44 DATASET_FOLDER = "HCP" TYPE = "single_direction" EXP_PATH = "~" LABELS_FILENAME = "bundle_peaks" FEATURES_FILENAME = "270g_125mm_peaks" DATASET = "HCP" RESOLUTION = "1.25mm" LABELS_TYPE = np.float32 HP.TRAIN_SUBJECTS, HP.VALIDATE_SUBJECTS, HP.TEST_SUBJECTS = ExpUtils.get_cv_fold( HP.CV_FOLD) num_batches_base = 5000 num_batches = { "train": num_batches_base, "validate": int(num_batches_base / 3.), "test": int(num_batches_base / 3.), } if custom_type is None: types = ["train", "validate", "test"] else: types = [custom_type] for type in types: dataManager = DataManagerTrainingNiftiImgs(HP) batch_gen = dataManager.get_batches( batch_size=HP.BATCH_SIZE, type=type, subjects=getattr(HP, type.upper() + "_SUBJECTS"), num_batches=num_batches[type]) for idx, batch in enumerate(batch_gen): print("Processing: {}".format(idx)) # DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_Y" # DATASET_DIR = "HCP_batches/All_sizes_DAug_XYZ" DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_XYZ" ExpUtils.make_dir(join(C.HOME, DATASET_DIR, type)) data = nib.Nifti1Image( batch["data"], ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION)) nib.save( data, join(C.HOME, DATASET_DIR, type, "batch_" + str(idx) + "_data.nii.gz")) seg = nib.Nifti1Image( batch["seg"], ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION)) nib.save( seg, join(C.HOME, DATASET_DIR, type, "batch_" + str(idx) + "_seg.nii.gz"))
def save_multilabel_img_as_multiple_files_endings(HP, img, affine, path): bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:] for idx, bundle in enumerate(bundles): img_seg = nib.Nifti1Image(img[:,:,:,idx], affine) ExpUtils.make_dir(join(path, "endings_segmentations")) nib.save(img_seg, join(path, "endings_segmentations", bundle + ".nii.gz"))