def save_multilabel_img_as_multiple_files_endings(HP, img, affine, path):
bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:]
for idx, bundle in enumerate(bundles):
img_seg = nib.Nifti1Image(img[:, :, :, idx], affine)
ExpUtils.make_dir(join(path, "endings_segmentations"))
nib.save(img_seg,
join(path, "endings_segmentations", bundle + ".nii.gz"))
def save_fusion_nifti_as_npy():
#Can leave this always the same (for 270g and 32g)
class HP:
DATASET = "HCP"
RESOLUTION = "1.25mm"
FEATURES_FILENAME = "270g_125mm_peaks"
LABELS_TYPE = np.int16
LABELS_FILENAME = "bundle_masks"
DATASET_FOLDER = "HCP"
#change this for 270g and 32g
DIFFUSION_FOLDER = "32g_25mm"
subjects = get_all_subjects()
# fold0 = ['687163', '685058', '683256', '680957', '679568', '677968', '673455', '672756', '665254', '654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469']
# fold1 = ['992774', '991267', '987983', '984472', '983773', '979984', '978578', '965771', '965367', '959574', '958976', '957974', '951457', '932554', '930449', '922854', '917255', '912447', '910241', '907656', '904044']
# fold2 = ['901442', '901139', '901038', '899885', '898176', '896879', '896778', '894673', '889579', '887373', '877269', '877168', '872764', '872158', '871964', '871762', '865363', '861456', '859671', '857263', '856766']
# fold3 = ['849971', '845458', '837964', '837560', '833249', '833148', '826454', '826353', '816653', '814649', '802844', '792766', '792564', '789373', '786569', '784565', '782561', '779370', '771354', '770352', '765056']
# fold4 = ['761957', '759869', '756055', '753251', '751348', '749361', '748662', '748258', '742549', '734045', '732243', '729557', '729254', '715647', '715041', '709551', '705341', '704238', '702133', '695768', '690152']
# subjects = fold2 + fold3 + fold4
# subjects = ['654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469']
print("\n\nProcessing Data...")
for s in subjects:
print("processing data subject {}".format(s))
start_time = time.time()
data = nib.load(
join(C.NETWORK_DRIVE, "HCP_fusion_" + DIFFUSION_FOLDER,
s + "_probmap.nii.gz")).get_data()
print("Done Loading")
data = np.nan_to_num(data)
data = DatasetUtils.scale_input_to_unet_shape(
data, HP.DATASET, HP.RESOLUTION)
data = data[:-1, :, :
-1, :] # cut one pixel at the end, because in scale_input_to_world_shape we ouputted 146 -> one too much at the end
ExpUtils.make_dir(
join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s))
np.save(
join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s,
DIFFUSION_FOLDER + "_xyz.npy"), data)
print("Took {}s".format(time.time() - start_time))
print("processing seg subject {}".format(s))
start_time = time.time()
# seg = ImgUtils.create_multilabel_mask(HP, s, labels_type=HP.LABELS_TYPE)
seg = nib.load(
join(C.NETWORK_DRIVE, "HCP_for_training_COPY", s,
HP.LABELS_FILENAME + ".nii.gz")).get_data()
if HP.RESOLUTION == "2.5mm":
seg = ImgUtils.resize_first_three_dims(seg, order=0, zoom=0.5)
seg = DatasetUtils.scale_input_to_unet_shape(
seg, HP.DATASET, HP.RESOLUTION)
np.save(
join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s,
"bundle_masks.npy"), seg)
print("Took {}s".format(time.time() - start_time))
def precompute_batches(custom_type=None):
'''
9000 slices per epoch -> 200 batches (batchsize=44) per epoch
=> 200-1000 batches needed
270g_125mm_bundle_peaks_Y: no DAug, no Norm, only Y
All_sizes_DAug_XYZ: 12g, 90g, 270g, DAug (no rotation, no elastic deform), Norm, XYZ
270g_125mm_bundle_peaks_XYZ: no DAug, Norm, XYZ
'''
class HP:
NORMALIZE_DATA = True
DATA_AUGMENTATION = False
CV_FOLD = 0
INPUT_DIM = (144, 144)
BATCH_SIZE = 44
DATASET_FOLDER = "HCP"
TYPE = "single_direction"
EXP_PATH = "~"
LABELS_FILENAME = "bundle_peaks"
FEATURES_FILENAME = "270g_125mm_peaks"
DATASET = "HCP"
RESOLUTION = "1.25mm"
LABELS_TYPE = np.float32
HP.TRAIN_SUBJECTS, HP.VALIDATE_SUBJECTS, HP.TEST_SUBJECTS = ExpUtils.get_cv_fold(HP.CV_FOLD)
num_batches_base = 5000
num_batches = {
"train": num_batches_base,
"validate": int(num_batches_base / 3.),
"test": int(num_batches_base / 3.),
}
if custom_type is None:
types = ["train", "validate", "test"]
else:
types = [custom_type]
for type in types:
dataManager = DataManagerTrainingNiftiImgs(HP)
batch_gen = dataManager.get_batches(batch_size=HP.BATCH_SIZE, type=type,
subjects=getattr(HP, type.upper() + "_SUBJECTS"), num_batches=num_batches[type])
for idx, batch in enumerate(batch_gen):
print("Processing: {}".format(idx))
# DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_Y"
# DATASET_DIR = "HCP_batches/All_sizes_DAug_XYZ"
DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_XYZ"
ExpUtils.make_dir(join(C.HOME, DATASET_DIR, type))
data = nib.Nifti1Image(batch["data"], ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION))
nib.save(data, join(C.HOME, DATASET_DIR, type, "batch_" + str(idx) + "_data.nii.gz"))
seg = nib.Nifti1Image(batch["seg"], ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION))
nib.save(seg, join(C.HOME, DATASET_DIR, type, "batch_" + str(idx) + "_seg.nii.gz"))
def save_multilabel_img_as_multiple_files_peaks(HP, img, affine, path):
bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:]
for idx, bundle in enumerate(bundles):
data = img[:, :, :, (idx * 3):(idx * 3) + 3]
if HP.FLIP_OUTPUT_PEAKS:
data[:, :, :, 2] *= -1 # flip z Axis for correct view in MITK
filename = bundle + "_f.nii.gz"
else:
filename = bundle + ".nii.gz"
img_seg = nib.Nifti1Image(data, affine)
ExpUtils.make_dir(join(path, "TOM"))
nib.save(img_seg, join(path, "TOM", filename))
def save_multilabel_img_as_multiple_files_peaks(HP, img, affine, path):
bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:]
for idx, bundle in enumerate(bundles):
data = img[:, :, :, (idx*3):(idx*3)+3]
if HP.FLIP_OUTPUT_PEAKS:
data[:, :, :, 2] *= -1 # flip z Axis for correct view in MITK
filename = bundle + "_f.nii.gz"
else:
filename = bundle + ".nii.gz"
img_seg = nib.Nifti1Image(data, affine)
ExpUtils.make_dir(join(path, "TOM"))
nib.save(img_seg, join(path, "TOM", filename))
def save_fusion_nifti_as_npy():
#Can leave this always the same (for 270g and 32g)
class HP:
DATASET = "HCP"
RESOLUTION = "1.25mm"
FEATURES_FILENAME = "270g_125mm_peaks"
LABELS_TYPE = np.int16
LABELS_FILENAME = "bundle_masks"
DATASET_FOLDER = "HCP"
#change this for 270g and 32g
DIFFUSION_FOLDER = "32g_25mm"
subjects = get_all_subjects()
# fold0 = ['687163', '685058', '683256', '680957', '679568', '677968', '673455', '672756', '665254', '654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469']
# fold1 = ['992774', '991267', '987983', '984472', '983773', '979984', '978578', '965771', '965367', '959574', '958976', '957974', '951457', '932554', '930449', '922854', '917255', '912447', '910241', '907656', '904044']
# fold2 = ['901442', '901139', '901038', '899885', '898176', '896879', '896778', '894673', '889579', '887373', '877269', '877168', '872764', '872158', '871964', '871762', '865363', '861456', '859671', '857263', '856766']
# fold3 = ['849971', '845458', '837964', '837560', '833249', '833148', '826454', '826353', '816653', '814649', '802844', '792766', '792564', '789373', '786569', '784565', '782561', '779370', '771354', '770352', '765056']
# fold4 = ['761957', '759869', '756055', '753251', '751348', '749361', '748662', '748258', '742549', '734045', '732243', '729557', '729254', '715647', '715041', '709551', '705341', '704238', '702133', '695768', '690152']
# subjects = fold2 + fold3 + fold4
# subjects = ['654754', '645551', '644044', '638049', '627549', '623844', '622236', '620434', '613538', '601127', '599671', '599469']
print("\n\nProcessing Data...")
for s in subjects:
print("processing data subject {}".format(s))
start_time = time.time()
data = nib.load(join(C.NETWORK_DRIVE, "HCP_fusion_" + DIFFUSION_FOLDER, s + "_probmap.nii.gz")).get_data()
print("Done Loading")
data = np.nan_to_num(data)
data = DatasetUtils.scale_input_to_unet_shape(data, HP.DATASET, HP.RESOLUTION)
data = data[:-1, :, :-1, :] # cut one pixel at the end, because in scale_input_to_world_shape we ouputted 146 -> one too much at the end
ExpUtils.make_dir(join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s))
np.save(join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s, DIFFUSION_FOLDER + "_xyz.npy"), data)
print("Took {}s".format(time.time() - start_time))
print("processing seg subject {}".format(s))
start_time = time.time()
# seg = ImgUtils.create_multilabel_mask(HP, s, labels_type=HP.LABELS_TYPE)
seg = nib.load(join(C.NETWORK_DRIVE, "HCP_for_training_COPY", s, HP.LABELS_FILENAME + ".nii.gz")).get_data()
if HP.RESOLUTION == "2.5mm":
seg = ImgUtils.resize_first_three_dims(seg, order=0, zoom=0.5)
seg = DatasetUtils.scale_input_to_unet_shape(seg, HP.DATASET, HP.RESOLUTION)
np.save(join(C.NETWORK_DRIVE, "HCP_fusion_npy_" + DIFFUSION_FOLDER, s, "bundle_masks.npy"), seg)
print("Took {}s".format(time.time() - start_time))
def copy_training_files_to_ssd(HP, data_path):
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
target_data_path = join("/ssd/", "tmp_" + id_generator(), HP.DATASET_FOLDER)
ExpUtils.make_dir(join(target_data_path))
#get all folders in data_path directory
subjects = [os.path.basename(os.path.normpath(d)) for d in glob(data_path + "/*/")]
for subject in subjects:
src = join(data_path, subject, HP.FEATURES_FILENAME)
target = join(target_data_path, subject, HP.FEATURES_FILENAME)
print("cp: {} -> {}".format(src, target))
# shutil.copyfile(src, target)
return target_data_path
def copy_training_files_to_ssd(HP, data_path):
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
target_data_path = join("/ssd/", "tmp_" + id_generator(),
HP.DATASET_FOLDER)
ExpUtils.make_dir(join(target_data_path))
#get all folders in data_path directory
subjects = [
os.path.basename(os.path.normpath(d))
for d in glob(data_path + "/*/")
]
for subject in subjects:
src = join(data_path, subject, HP.FEATURES_FILENAME)
target = join(target_data_path, subject, HP.FEATURES_FILENAME)
print("cp: {} -> {}".format(src, target))
# shutil.copyfile(src, target)
return target_data_path
def save_multilabel_img_as_multiple_files_endings_OLD(HP, img, affine, path, multilabel=True):
'''
multilabel True: save as 1 and 2 without fourth dimension
multilabel False: save with beginnings and endings combined
'''
# bundles = ExpUtils.get_bundle_names("20")[1:]
bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:]
for idx, bundle in enumerate(bundles):
data = img[:, :, :, (idx * 2):(idx * 2) + 2] > 0
multilabel_img = np.zeros(data.shape[:3])
if multilabel:
multilabel_img[data[:, :, :, 0]] = 1
multilabel_img[data[:, :, :, 1]] = 2
else:
multilabel_img[data[:, :, :, 0]] = 1
multilabel_img[data[:, :, :, 1]] = 1
img_seg = nib.Nifti1Image(multilabel_img, affine)
ExpUtils.make_dir(join(path, "endings"))
nib.save(img_seg, join(path, "endings", bundle + ".nii.gz"))
def save_multilabel_img_as_multiple_files_endings_OLD(HP, img, affine, path, multilabel=True):
'''
multilabel True: save as 1 and 2 without fourth dimension
multilabel False: save with beginnings and endings combined
'''
# bundles = ExpUtils.get_bundle_names("20")[1:]
bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:]
for idx, bundle in enumerate(bundles):
data = img[:, :, :, (idx * 2):(idx * 2) + 2] > 0
multilabel_img = np.zeros(data.shape[:3])
if multilabel:
multilabel_img[data[:, :, :, 0]] = 1
multilabel_img[data[:, :, :, 1]] = 2
else:
multilabel_img[data[:, :, :, 0]] = 1
multilabel_img[data[:, :, :, 1]] = 1
img_seg = nib.Nifti1Image(multilabel_img, affine)
ExpUtils.make_dir(join(path, "endings"))
nib.save(img_seg, join(path, "endings", bundle + ".nii.gz"))
def precompute_batches(custom_type=None):
'''
9000 slices per epoch -> 200 batches (batchsize=44) per epoch
=> 200-1000 batches needed
270g_125mm_bundle_peaks_Y: no DAug, no Norm, only Y
All_sizes_DAug_XYZ: 12g, 90g, 270g, DAug (no rotation, no elastic deform), Norm, XYZ
270g_125mm_bundle_peaks_XYZ: no DAug, Norm, XYZ
'''
class HP:
NORMALIZE_DATA = True
DATA_AUGMENTATION = False
CV_FOLD = 0
INPUT_DIM = (144, 144)
BATCH_SIZE = 44
DATASET_FOLDER = "HCP"
TYPE = "single_direction"
EXP_PATH = "~"
LABELS_FILENAME = "bundle_peaks"
FEATURES_FILENAME = "270g_125mm_peaks"
DATASET = "HCP"
RESOLUTION = "1.25mm"
LABELS_TYPE = np.float32
HP.TRAIN_SUBJECTS, HP.VALIDATE_SUBJECTS, HP.TEST_SUBJECTS = ExpUtils.get_cv_fold(
HP.CV_FOLD)
num_batches_base = 5000
num_batches = {
"train": num_batches_base,
"validate": int(num_batches_base / 3.),
"test": int(num_batches_base / 3.),
}
if custom_type is None:
types = ["train", "validate", "test"]
else:
types = [custom_type]
for type in types:
dataManager = DataManagerTrainingNiftiImgs(HP)
batch_gen = dataManager.get_batches(
batch_size=HP.BATCH_SIZE,
type=type,
subjects=getattr(HP,
type.upper() + "_SUBJECTS"),
num_batches=num_batches[type])
for idx, batch in enumerate(batch_gen):
print("Processing: {}".format(idx))
# DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_Y"
# DATASET_DIR = "HCP_batches/All_sizes_DAug_XYZ"
DATASET_DIR = "HCP_batches/270g_125mm_bundle_peaks_XYZ"
ExpUtils.make_dir(join(C.HOME, DATASET_DIR, type))
data = nib.Nifti1Image(
batch["data"],
ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION))
nib.save(
data,
join(C.HOME, DATASET_DIR, type,
"batch_" + str(idx) + "_data.nii.gz"))
seg = nib.Nifti1Image(
batch["seg"],
ImgUtils.get_dwi_affine(HP.DATASET, HP.RESOLUTION))
nib.save(
seg,
join(C.HOME, DATASET_DIR, type,
"batch_" + str(idx) + "_seg.nii.gz"))
def save_multilabel_img_as_multiple_files_endings(HP, img, affine, path):
bundles = ExpUtils.get_bundle_names(HP.CLASSES)[1:]
for idx, bundle in enumerate(bundles):
img_seg = nib.Nifti1Image(img[:,:,:,idx], affine)
ExpUtils.make_dir(join(path, "endings_segmentations"))
nib.save(img_seg, join(path, "endings_segmentations", bundle + ".nii.gz"))