def find_bval_bvecs(subjectpath, subject=""):
fbtable = glob.glob(os.path.join(subjectpath, "b_table.txt"))
fbvals = glob.glob(os.path.join(subjectpath, "input_bvals.txt"))
fbvecs = glob.glob(os.path.join(subjectpath, "*input_gradient_matrix*"))
bxhs = glob.glob(os.path.join(subjectpath, "*.bxh*"))
fbvals_txt = glob.glob(os.path.join(subjectpath, "*bvals.txt"))
if np.size(fbtable) > 0:
fbvals = []
fbvecs = []
fbtable = fbtable[0]
with open(fbtable, 'rb') as source:
for line in source:
bvals_all = str(line).split("'")[1]
bvals_all = bvals_all.split("\\n")[0]
bvals_all = bvals_all.split("\\t")
fbvals.append(float(bvals_all[0]))
fbvecs.append([
float(bvals_all[1]),
float(bvals_all[2]),
float(bvals_all[3])
])
#return np.array(bvals), np.array(bvecs)
fbvals = np.array(fbvals)
fbvecs = np.array(fbvecs)
elif np.size(fbvals) > 0 and np.size(fbvecs) > 0:
fbvals = fbvals[0]
fbvecs = fbvecs[0]
elif np.size(bxhs) > 0:
dwipath = largerfile(
subjectpath, identifier=".nii"
) #This just catches the LARGEST file, which should be a dwi file. This is obviously an unstable method and the best way to handle it would be to go through every bxh file and go through them individualls
bxhpath = dwipath.replace(".nii.gz", ".bxh")
bxhpath = bxhpath.replace(".nii", ".bxh")
fbvals, fbvecs, _, _, _, _ = extractbvec_fromheader(
bxhpath,
fileoutpath=os.path.join(subjectpath, subject),
save="all")
elif np.size(fbvals_txt) > 0:
fbvecs_txt = glob.glob(os.path.join(subjectpath, "*bvec*.txt"))
if np.size(fbvals_txt) == 1 and np.size(fbvecs_txt) == 1:
fbvals, fbvecs = read_bval_bvec(fbvals_txt[0], fbvecs_txt[0])
else:
raise Exception('Too many possible bvalue files in folder')
else:
raise Exception(
"Sorry, nothing here looks like it could serve as a way to extract bvalues"
)
return fbvals, fbvecs
def launch_preprocessing(subj, raw_nii, outpath, cleanup=False, nominal_bval=4000, SAMBA_inputs_folder=None,
shortcuts_all_folder = None, gunniespath="~/gunnies/", processes=1, masking="bet", ref=None,
transpose=None, overwrite=False, denoise='None', recenter=0, verbose=False):
proc_name ="diffusion_prep_" # Not gonna call it diffusion_calc so we don't assume it does the same thing as the civm pipeline
work_dir=os.path.join(outpath,proc_name+subj)
"""
for filePath in glob.glob(os.path.join(work_dir,'*')):
modTimesinceEpoc = os.path.getmtime(filePath)
modificationTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(modTimesinceEpoc))
if modificationTime[5:7]=='09' and int(modificationTime[8:10])>9:
os.remove(filePath)
"""
if verbose:
print(f"Processing diffusion data with runno/subj: {subj}")
print(f"Work directory is {work_dir}")
mkcdir(work_dir)
sbatch_folder =os.path.join(work_dir,"sbatch")
mkcdir(sbatch_folder)
#nii_path = os.path.join(work_dir,'nii4D_'+subj + '.nii.gz')
#if not os.path.exists(nii_path):
# shutil.copy(raw_nii, nii_path)
nii_name=os.path.split(raw_nii)[1]
niifolder = os.path.dirname(raw_nii)
ext = ".nii.gz"
nii_ext=getext(nii_name)
bxheader = nii_name.replace(nii_ext,".bxh")
bxheader = os.path.join(niifolder, bxheader)
bvecs = os.path.join(work_dir, subj+"_bvecs.txt")
bvals =bvecs.replace("bvecs","bvals")
if verbose:
print(f"Original nifti is at {nii_name}\nbvecs are at {bvecs}\nbvals are at {bvals}\n")
if not os.path.exists(bvecs):
if verbose:
print("Extracting diff directions")
#print("Bvals and bvecs not found, using extractdiffdirs, however it it NOT RELIABLE, beware!")
bvec_cmd = (f"extractdiffdirs --colvectors --writebvals --fieldsep='\t' --space=RAI {bxheader} {bvecs} {bvals}")
os.system(bvec_cmd)
# Make dwi for mask generation purposes.
tmp_mask = os.path.join(work_dir,f"{subj}_tmp_mask{ext}")
raw_dwi = os.path.join(work_dir,f"{subj}_raw_dwi.nii.gz")
orient_string = os.path.join(work_dir,"relative_orientation.txt")
if shortcuts_all_folder is not None:
#nii_path_link = os.path.join(shortcuts_all_folder, f"{subj}_rawnii{ext}")
#if not os.path.exists(nii_path_link) or overwrite:
# buildlink(nii_path, nii_path_link)
bvecs_new = os.path.join(shortcuts_all_folder, subj + "_bvecs.txt")
bvals_new = os.path.join(shortcuts_all_folder, subj + "_bvals.txt")
if not os.path.exists(bvecs_new) or not os.path.exists(bvals_new) or overwrite:
shutil.copyfile(bvecs, bvecs_new)
shutil.copyfile(bvals, bvals_new)
final_mask = os.path.join(work_dir, f'{subj}_mask{ext}')
#if (not os.path.exists(final_mask) and not os.path.exists(tmp_mask)) or overwrite:
if not os.path.exists(tmp_mask) or overwrite:
if not os.path.exists(raw_dwi) or overwrite:
select_cmd = f"select_dwi_vols {raw_nii} {bvals} {raw_dwi} {nominal_bval} -m"
os.system(select_cmd)
if not os.path.exists(tmp_mask) or overwrite:
if 'median' in masking:
tmp = tmp_mask.replace("_mask", "")
if np.size(masking.split('_'))>1:
median_radius = int(masking.split('_')[1])
else:
median_radius = 4
median_mask_make(raw_dwi, tmp, outpathmask=tmp_mask, median_radius = median_radius, numpass=median_radius)
elif masking=="bet":
tmp=tmp_mask.replace("_mask", "")
bet_cmd = f"bet {raw_dwi} {tmp} -m -n -R"
os.system(bet_cmd)
else:
raise Exception("Unrecognized masking type")
# I think this part is done later more properly: if create_subj_space_files: for contrast in ['dwi', 'b0', 'mask']:
#if SAMBA_inputs_folder is not None:
# mask_subj_link = os.path.join(SAMBA_inputs_folder,f'{subj}_subjspace_mask{ext}')
# if not os.path.exists(mask_subj_link) or overwrite:
# shutil.copy(tmp_mask, mask_subj_link)
#if cleanup and (os.path.exists(tmp_mask) and os.path.exists(raw_dwi)):
# os.remove(raw_dwi)
#overwrite=False
# Run Local PCA Denoising algorithm on 4D nifti:
masked_nii = os.path.join(work_dir, nii_name)
if not "nii.gz" in masked_nii:
masked_nii = masked_nii.replace(".nii", ".nii.gz")
masked_nii = masked_nii.replace(ext, "_masked" + ext)
if denoise.lower()=='lpca':
D_subj=f'LPCA_{subj}';
elif denoise.lower()=='mpca':
D_subj=f'MPCA_{subj}';
elif denoise=="None" or denoise is None:
D_subj = f'{subj}'
if denoise=="None" or denoise is None:
denoised_nii = masked_nii
if not os.path.exists(masked_nii) or overwrite:
fsl_cmd = f"fslmaths {raw_nii} -mas {tmp_mask} {masked_nii} -odt 'input'";
os.system(fsl_cmd)
else:
denoised_nii = os.path.join(work_dir,f"{D_subj}_nii4D.nii.gz")
if not os.path.exists(denoised_nii) or overwrite:
if not os.path.exists(masked_nii) or overwrite:
fsl_cmd = f"fslmaths {raw_nii} -mas {tmp_mask} {masked_nii} -odt 'input'";
os.system(fsl_cmd)
basic_LPCA_denoise_func(subj,masked_nii,bvecs,denoised_nii, processes=processes,
denoise=denoise, verbose=False) #to improve and make multiprocessing
#if cleanup and os.path.exists(denoised_nii) and os.path.exists(masked_nii) and denoised_nii!=masked_nii:
# os.remove(masked_nii)
# Run coregistration/eddy current correction:
coreg_nii_old = f'{outpath}/co_reg_{D_subj}_m00-results/Reg_{D_subj}_nii4D{ext}';
coreg_nii = os.path.join(work_dir,f'Reg_{D_subj}_nii4D{ext}')
if not cleanup:
coreg_nii=coreg_nii_old
if not os.path.exists(coreg_nii) or overwrite:
if not os.path.exists(coreg_nii_old) or overwrite:
temp_cmd = os.path.join(gunniespath,'co_reg_4d_stack_tmpnew.bash')+f' {denoised_nii} {D_subj} 0 {outpath} 0';
os.system(temp_cmd)
if cleanup:
shutil.move(coreg_nii_old,coreg_nii)
if shortcuts_all_folder is not None:
coreg_link = os.path.join(shortcuts_all_folder,f'{subj}_subjspace_coreg{ext}')
if not os.path.exists(coreg_link) or overwrite:
buildlink(coreg_nii, coreg_link)
toeddy=False
if toeddy:
#fsl_cmd = f"fslmaths {raw_nii} -mas {tmp_mask} {masked_nii} -odt 'input'";
#os.system(fsl_cmd)
eddy_cmd = f"eddy --imain={coreg_nii} --mask={tmp_mask} --acqp=acq_params.txt --index={os.path.join(work_dir,'index.txt')} --bvecs={bvecs} --bvals={bvals} --topup=topup_results --repol --out = {os.path.join(work_dir,f'Reg_{D_subj}_nii4D_eddy{ext}')}"
os.system(eddy_cmd)
coreg_inputs=os.path.join(outpath,f'co_reg_{D_subj}_m00-inputs')
coreg_work=coreg_inputs.replace('-inputs','-work')
coreg_results=coreg_inputs.replace('-inputs','-results')
if cleanup and os.path.exists(coreg_nii) and os.path.isdir(coreg_inputs):
shutil.rmtree(coreg_inputs)
if cleanup and os.path.exists(coreg_nii) and os.path.isdir(coreg_work):
shutil.rmtree(coreg_work)
if cleanup and os.path.exists(coreg_nii) and os.path.isdir(coreg_results):
shutil.rmtree(coreg_results)
# Generate tmp DWI:
tmp_dwi_out=os.path.join(work_dir, f'{subj}_tmp_dwi{ext}')
dwi_out=os.path.join(work_dir,f'{subj}_dwi{ext}')
if not os.path.exists(tmp_dwi_out) or overwrite:
cmd=f'select_dwi_vols {coreg_nii} {bvals} {tmp_dwi_out} {nominal_bval} -m'
os.system(cmd)
# Generate tmp B0:
tmp_b0_out=os.path.join(work_dir,f'{subj}_tmp_b0{ext}')
b0_out = os.path.join(work_dir, f'{subj}_b0{ext}')
if (not os.path.exists(b0_out) and not os.path.exists(tmp_b0_out)) or overwrite:
cmd=f'select_dwi_vols {coreg_nii} {bvals} {tmp_b0_out} 0 -m;'
os.system(cmd)
#overwrite=False
#elif cleanup and os.path.exists(tmp_b0_out):
# os.remove(tmp_b0_out)
# Generate DTI contrasts and perform some tracking QA:
if cleanup:
c_string=' --cleanup '
else:
c_string=''
#Important note: this is what first creates the fa, md, etc
if len(glob.glob(os.path.join(work_dir,f'*.fib.gz.md{ext}'))) == 0 or overwrite:
if overwrite:
oldfiles = glob.glob(os.path.join(work_dir, f'*.fib.gz*'))
for oldfile in oldfiles:
os.remove(oldfile)
cmd = 'bash ' + os.path.join(gunniespath,'dti_qa_with_dsi_studio_weirdcall.bash')+f' {coreg_nii} {bvecs} {tmp_mask} {work_dir} {c_string}';
os.system(cmd)
#Save the subject space dti results
#Generate tmp MD:
for contrast in ['md']:
real_file=largerfile(os.path.join(work_dir,f'*.fib.gz.{contrast}{ext}')) #Catch the 'real file' for each contrast
tmp_file = f'{work_dir}/{subj}_tmp_{contrast}{ext}';
if not os.path.exists(tmp_file):
shutil.copy(real_file,tmp_file)
tmp_md = f'{work_dir}/{subj}_tmp_md{ext}';
if ref=="md" or ref is None:
reference=tmp_md
elif ref=="coreg":
reference=coreg_nii
elif os.path.exists(ref):
reference=ref
reference_file = os.path.join(work_dir, f'{subj}_reference{ext}')
if not os.path.exists(reference_file):
shutil.copy(reference, reference_file)
if shortcuts_all_folder is not None:
bonus_ref_link = os.path.join(shortcuts_all_folder, f'{subj}_reference{ext}')
if not os.path.exists(bonus_ref_link) or overwrite:
buildlink(reference_file,bonus_ref_link)
#give new header to the non-dti files using md as reference
for contrast in ['dwi', 'b0', 'mask']:
tmp_file=os.path.join(work_dir,f'{subj}_tmp_{contrast}{ext}')
tmp2_file=os.path.join(work_dir,f'{subj}_tmp2_{contrast}{ext}')
final_file=os.path.join(work_dir,f'{subj}_{contrast}{ext}')
if ((not os.path.exists(tmp2_file) and not os.path.exists(final_file)) or overwrite):
if not os.path.exists(tmp_file):
raise Exception("Tmp file was not created, need to rerun previous processes")
else:
header_superpose(reference, tmp_file, outpath=tmp2_file)
create_subj_space_files = True
if create_subj_space_files:
for contrast in ['dwi', 'b0', 'mask']:
tmp_file = os.path.join(work_dir, f'{subj}_tmp_{contrast}{ext}')
subj_file = os.path.join(work_dir, f'{subj}_subjspace_{contrast}{ext}')
if not os.path.exists(subj_file) or overwrite:
if not os.path.exists(tmp_file):
raise Exception("Tmp file was not created, need to rerun previous processes")
else:
header_superpose(raw_dwi, tmp_file, outpath=subj_file)
if shortcuts_all_folder is not None:
subj_link = os.path.join(shortcuts_all_folder, f'{subj}_subjspace_{contrast}{ext}')
if not os.path.exists(subj_link) or overwrite:
buildlink(subj_file, subj_link)
#write the relative orientation file here
if not os.path.isfile(orient_string) or overwrite:
if os.path.isfile(orient_string):
os.remove(orient_string)
file = os.path.join(work_dir,subj+'_tmp_mask'+ext);
cmd = 'bash ' + os.path.join(gunniespath,'find_relative_orientation_by_CoM.bash') + f' {reference_file} {file}'
orient_relative = subprocess.getoutput(cmd)
with open(orient_string, 'w') as f:
f.write(orient_relative)
else:
orient_relative = open(orient_string, mode='r').read()
if SAMBA_inputs_folder is not None:
subj_orient_string = os.path.join(SAMBA_inputs_folder, f'{subj}_relative_orientation.txt')
shutil.copy(orient_string, subj_orient_string)
if shortcuts_all_folder is not None:
subj_orient_string = os.path.join(shortcuts_all_folder, f'{subj}_relative_orientation.txt')
shutil.copy(orient_string, subj_orient_string)
#check extracted values from relative orientation vals
orientation_out = orient_relative.split(',')[0]
orientation_out = orientation_out.split(':')[1]
orientation_in = orient_relative.split(',')[1]
orientation_in = orientation_in.split(':')[1]
if verbose:
print(f'flexible orientation: {orientation_in}');
print(f'reference orientation: {orientation_out}');
#apply the orientation modification to specified contrasts
for contrast in ['dwi', 'b0', 'mask']:
img_in=os.path.join(work_dir,f'{subj}_tmp2_{contrast}{ext}')
img_out=os.path.join(work_dir,f'{subj}_{contrast}{ext}')
if not os.path.isfile(img_out) or overwrite:
if orientation_out != orientation_in:
print('TRYING TO REORIENT...b0 and dwi and mask')
if os.path.exists(img_in) and (not os.path.exists(img_out) or overwrite):
img_transform_exec(img_in, orientation_in, orientation_out, img_out)
if os.path.exists(img_out):
os.remove(img_in)
elif os.path.exists(img_out) and cleanup:
os.remove(img_in)
else:
shutil.move(img_in,img_out)
if SAMBA_inputs_folder is not None:
inputs_space_link = os.path.join(SAMBA_inputs_folder, f'{subj}_{contrast}{ext}')
if not os.path.exists(inputs_space_link) or overwrite:
buildlink(img_out, inputs_space_link)
if shortcuts_all_folder is not None:
inputs_space_link = os.path.join(shortcuts_all_folder, f'{subj}_{contrast}{ext}')
if not os.path.exists(inputs_space_link) or overwrite:
buildlink(img_out, inputs_space_link)
mask = os.path.join(work_dir,f'{subj}_mask{ext}')
b0 = os.path.join(work_dir,f'{subj}_b0{ext}')
#if cleanup and os.path.exists(dwi_out) and os.path.exists(tmp_dwi_out):
# os.remove(tmp_dwi_out)
for contrast in ['fa0', 'rd', 'ad', 'md']:
real_file=largerfile(os.path.join(work_dir,f'*.fib.gz.{contrast}{ext}')) # It will be fun times if we ever have more than one match to this pattern...
#inputspace = real_file
inputspace = os.path.join(work_dir, f'{subj}_inputspace_{contrast}{ext}')
contrast=contrast.replace('0','')
#linked_file=os.path.join(shortcut_dir,f'{subj}_{contrast}{ext}')
linked_file_w=os.path.join(work_dir,f'{subj}_{contrast}{ext}')
made_newfile = affine_superpose(dwi_out, real_file, outpath = inputspace, transpose=transpose)
if not made_newfile:
inputspace = real_file
if not os.path.isfile(linked_file_w) or overwrite:
buildlink(inputspace, linked_file_w)
if SAMBA_inputs_folder is not None:
#warnings.warn('should reach this!')
blinked_file = os.path.join(SAMBA_inputs_folder, f'{subj}_{contrast}{ext}')
if not os.path.exists(blinked_file) or overwrite:
buildlink(inputspace, blinked_file)
print(f'build link from {inputspace} to {blinked_file}')
if shortcuts_all_folder is not None:
#warnings.warn('should reach this!')
blinked_file = os.path.join(shortcuts_all_folder, f'{subj}_{contrast}{ext}')
if not os.path.exists(blinked_file) or overwrite:
buildlink(inputspace, blinked_file)
print(f'build link from {inputspace} to {blinked_file}')
if create_subj_space_files:
for contrast in ['fa0', 'rd', 'ad', 'md']:
real_file = largerfile(os.path.join(work_dir,
f'*.fib.gz.{contrast}{ext}')) # It will be fun times if we ever have more than one match to this pattern...
contrast = contrast.replace('0', '')
subj_file_tmp = os.path.join(work_dir, f'{subj}_subjspace_tmp_{contrast}{ext}')
subj_file = os.path.join(work_dir, f'{subj}_subjspace_{contrast}{ext}')
if not os.path.exists(subj_file) or overwrite:
if not os.path.exists(subj_file_tmp):
if orientation_out != orientation_in:
print('TRYING TO REORIENT...b0 and dwi and mask')
if os.path.exists(real_file) and (not os.path.exists(subj_file) or overwrite):
img_transform_exec(real_file, orientation_out, orientation_in, subj_file_tmp)
else:
shutil.copy(real_file,subj_file_tmp)
header_superpose(raw_dwi, subj_file_tmp, outpath=subj_file)
if shortcuts_all_folder is not None:
subj_link = os.path.join(shortcuts_all_folder, f'{subj}_subjspace_{contrast}{ext}')
buildlink(subj_file, subj_link)
#if cleanup:
# tmp_files = glob.glob(os.path.join(work_dir, '*tmp*'))
# for file in tmp_files:
# os.remove(file)
nii_path = os.path.join(work_dir,'nii4D_'+subj + ext)
if os.path.exists(nii_path):
os.remove(nii_path)
if cleanup:
reg_src = os.path.join(work_dir,'Reg_' + subj + f'_nii4D{ext}.src.gz')
if os.path.exists(reg_src):
os.remove(reg_src)
reg_src_fib = os.path.join(work_dir,'Reg_' + subj + f'_nii4D{ext}.src.gz.dti.fib.gz')
if os.path.exists(reg_src):
os.remove(reg_src)
def launch_preprocessing(subj, raw_nii, outpath, cleanup=False, nominal_bval=4000, bonusshortcutfolder=None,
gunniespath="~/gunnies/", processes=1, masking="bet", ref=None, transpose=None,
overwrite=False, denoise='None', recenter=0, verbose=False):
proc_name ="diffusion_prep_" # Not gonna call it diffusion_calc so we don't assume it does the same thing as the civm pipeline
work_dir=os.path.join(outpath,proc_name+subj)
if verbose:
print(f"Processing diffusion data with runno/subj: {subj}")
print(f"Work directory is {work_dir}")
mkcdir(work_dir)
sbatch_folder =os.path.join(work_dir,"sbatch")
mkcdir(sbatch_folder)
nii_name=os.path.split(raw_nii)[1]
niifolder = os.path.dirname(raw_nii)
ext = ".nii.gz"
nii_ext=getext(nii_name)
bxheader = nii_name.replace(nii_ext,".bxh")
bxheader = os.path.join(niifolder, bxheader)
bvecs = os.path.join(work_dir, subj+"_bvecs.txt")
bvals =bvecs.replace("bvecs","bvals")
if verbose:
print(f"Original nifti is at {nii_name}\nbvecs are at {bvecs}\nbvals are at {bvals}\n")
if not os.path.exists(bvecs):
if verbose:
print("Extracting diff directions")
#print("Bvals and bvecs not found, using extractdiffdirs, however it it NOT RELIABLE, beware!")
bvec_cmd = (f"extractdiffdirs --colvectors --writebvals --fieldsep='\t' --space=RAI {bxheader} {bvecs} {bvals}")
os.system(bvec_cmd)
# Make dwi for mask generation purposes.
orient_string = os.path.join(work_dir,'relative_orientation.txt')
tmp_mask = os.path.join(work_dir,f"{subj}_tmp_dwi_mask{ext}")
raw_dwi = os.path.join(work_dir,f"{subj}_raw_dwi.nii.gz")
orient_string = os.path.join(work_dir,"relative_orientation.txt")
if bonusshortcutfolder is not None:
raw_nii_link = os.path.join(bonusshortcutfolder, f"{subj}_rawnii{ext}")
if not os.path.exists(raw_nii_link) or overwrite:
buildlink(raw_nii, raw_nii_link)
bvecs_new = os.path.join(bonusshortcutfolder,subj+"_bvecs.txt")
bvals_new = os.path.join(bonusshortcutfolder,subj+"_bvals.txt")
if not os.path.exists(bvecs_new) or not os.path.exists(bvals_new) or overwrite:
shutil.copyfile(bvecs,bvecs_new)
shutil.copyfile(bvals,bvals_new)
final_mask = os.path.join(work_dir, f'{subj}_dwi_mask{ext}')
if (not os.path.exists(final_mask) and not os.path.exists(tmp_mask)) or overwrite:
if not os.path.exists(raw_dwi) or overwrite:
select_cmd = f"select_dwi_vols {raw_nii} {bvals} {raw_dwi} {nominal_bval} -m"
os.system(select_cmd)
if not os.path.exists(tmp_mask) or overwrite:
if masking=="median":
tmp = tmp_mask.replace("_mask", "")
median_mask_make(raw_dwi, tmp, outpathmask=tmp_mask)
elif masking=="bet":
tmp=tmp_mask.replace("_mask", "")
bet_cmd = f"bet {raw_dwi} {tmp} -m -n -R"
os.system(bet_cmd)
else:
raise Exception("Unrecognized masking type")
mask_subjspace =
if bonusshortcutfolder is not None:
mask_subj_link = os.path.join(bonusshortcutfolder,f'{subj}_mask_subjspace{ext}')
if not os.path.exists(mask_subj_link) or overwrite:
buildlink(tmp_mask, mask_subj_link)
#if cleanup and (os.path.exists(tmp_mask) and os.path.exists(raw_dwi)):
# os.remove(raw_dwi)
#overwrite=False
# Run Local PCA Denoising algorithm on 4D nifti:
masked_nii = os.path.join(work_dir, nii_name)
if not "nii.gz" in masked_nii:
masked_nii = masked_nii.replace(".nii", ".nii.gz")
masked_nii = masked_nii.replace(ext, "_masked" + ext)
if denoise.lower()=='lpca':
D_subj=f'LPCA_{subj}';
elif denoise.lower()=='mpca':
D_subj=f'MPCA_{subj}';
elif denoise=="None" or denoise is None:
D_subj = f'{subj}'
if denoise=="None" or denoise is None:
denoised_nii = masked_nii
if not os.path.exists(masked_nii) or overwrite:
fsl_cmd = f"fslmaths {raw_nii} -mas {tmp_mask} {masked_nii} -odt 'input'";
os.system(fsl_cmd)
else:
denoised_nii = os.path.join(work_dir,f"{D_subj}_nii4D.nii.gz")
if not os.path.exists(denoised_nii) or overwrite:
if not os.path.exists(masked_nii) or overwrite:
fsl_cmd = f"fslmaths {raw_nii} -mas {tmp_mask} {masked_nii} -odt 'input'";
os.system(fsl_cmd)
basic_LPCA_denoise_func(subj,masked_nii,bvecs,denoised_nii, processes=processes,
denoise=denoise, verbose=False) #to improve and make multiprocessing
#if cleanup and os.path.exists(denoised_nii) and os.path.exists(masked_nii) and denoised_nii!=masked_nii:
# os.remove(masked_nii)
# Run coregistration/eddy current correction:
coreg_nii_old = f'{outpath}/co_reg_{D_subj}_m00-results/Reg_{D_subj}_nii4D{ext}';
coreg_nii = os.path.join(work_dir,f'Reg_{D_subj}_nii4D{ext}')
if not cleanup:
coreg_nii=coreg_nii_old
if not os.path.exists(coreg_nii) or overwrite:
if not os.path.exists(coreg_nii_old) or overwrite:
temp_cmd = os.path.join(gunniespath,'co_reg_4d_stack_tmpnew.bash')+f' {denoised_nii} {D_subj} 0 {outpath} 0';
os.system(temp_cmd)
if cleanup:
shutil.move(coreg_nii_old,coreg_nii)
if bonusshortcutfolder is not None:
coreg_link = os.path.join(bonusshortcutfolder,f'{subj}_coreg{ext}')
if not os.path.exists(coreg_link) or overwrite:
buildlink(coreg_nii, coreg_link)
toeddy=False
if toeddy:
#fsl_cmd = f"fslmaths {raw_nii} -mas {tmp_mask} {masked_nii} -odt 'input'";
#os.system(fsl_cmd)
eddy_cmd = f"eddy --imain={coreg_nii} --mask={tmp_mask} --acqp=acq_params.txt --index={os.path.join(work_dir,'index.txt')} --bvecs={bvecs} --bvals={bvals} --topup=topup_results --repol --out = {os.path.join(work_dir,f'Reg_{D_subj}_nii4D_eddy{ext}')}"
os.system(eddy_cmd)
coreg_inputs=os.path.join(outpath,f'co_reg_{D_subj}_m00-inputs')
coreg_work=coreg_inputs.replace('-inputs','-work')
coreg_results=coreg_inputs.replace('-inputs','-results')
if cleanup and os.path.exists(coreg_nii) and os.path.isdir(coreg_inputs):
shutil.rmtree(coreg_inputs)
if cleanup and os.path.exists(coreg_nii) and os.path.isdir(coreg_work):
shutil.rmtree(coreg_work)
if cleanup and os.path.exists(coreg_nii) and os.path.isdir(coreg_results):
shutil.rmtree(coreg_results)
# Generate tmp DWI:
tmp_dwi_out=os.path.join(work_dir, f'{subj}_tmp_dwi{ext}')
dwi_out=os.path.join(work_dir,f'{subj}_dwi{ext}')
if not os.path.exists(dwi_out) or overwrite:
if not os.path.exists(tmp_dwi_out) or overwrite:
cmd=f'select_dwi_vols {coreg_nii} {bvals} {tmp_dwi_out} {nominal_bval} -m'
os.system(cmd)
if bonusshortcutfolder is not None:
tmp_dwi_out_link = os.path.join(bonusshortcutfolder,f'{subj}_mask_subjspace{ext}')
if not os.path.exists(tmp_dwi_out_link) or overwrite:
buildlink(tmp_dwi_out, tmp_dwi_out_link)
# Generate tmp B0:
tmp_b0_out=os.path.join(work_dir,f'{subj}_tmp_b0{ext}')
b0_out = os.path.join(work_dir, f'{subj}_b0{ext}')
if (not os.path.exists(b0_out) and not os.path.exists(tmp_b0_out)) or overwrite:
cmd=f'select_dwi_vols {coreg_nii} {bvals} {tmp_b0_out} 0 -m;'
os.system(cmd)
#overwrite=False
#elif cleanup and os.path.exists(tmp_b0_out):
# os.remove(tmp_b0_out)
# Generate DTI contrasts and perform some tracking QA:
if cleanup:
c_string=' --cleanup '
else:
c_string=''
#Important note: this is what first creates the fa, md, etc
overwrite = True
if len(glob.glob(os.path.join(work_dir,f'*.fib.gz.md{ext}'))) == 0 or overwrite:
if overwrite:
os.remove(os.path.join(work_dir, f'*.fib.gz.*{ext}'))
cmd = os.path.join(gunniespath,'dti_qa_with_dsi_studio.bash')+f' {coreg_nii} {bvecs} {tmp_mask} {work_dir} {c_string}';
os.system(cmd)
overwrite = False
#Save the subject space dti results
#Generate tmp MD:
for contrast in ['md']:
real_file=largerfile(os.path.join(work_dir,f'*.fib.gz.{contrast}{ext}')) #Catch the 'real file' for each contrast
tmp_file = f'{work_dir}/{subj}_tmp_{contrast}{ext}';
#contrast=contrast.replace('0','') #some little nonsense mostly to deal with the 'fa0' name to become 'fa', probably sohould change it
if not os.path.exists(tmp_file):
shutil.copy(real_file,tmp_file)
tmp_md = f'{work_dir}/{subj}_tmp_md{ext}';
if ref=="md" or ref is None:
reference=tmp_md
elif ref=="coreg":
reference=coreg_nii
elif os.path.exists(ref):
reference=ref
reference_file = os.path.join(work_dir, f'{subj}_reference{ext}')
if not os.path.exists(reference_file):
shutil.copy(reference, reference_file)
if bonusshortcutfolder is not None:
bonus_ref_link = os.path.join(bonusshortcutfolder, f'{subj}_reference{ext}')
if not os.path.exists(bonus_ref_link) or overwrite:
buildlink(reference_file,bonus_ref_link)
"""
for contrast in ['md']:
real_file=largerfile(os.path.join(work_dir,f'*.fib.gz.{contrast}{ext}')) #Catch the 'real file' for each contrast
tmp_file = f'{work_dir}/{subj}_tmp_{contrast}{ext}';
#contrast=contrast.replace('0','') #some little nonsense mclassifierostly to deal with the 'fa0' name to become 'fa', probably sohould change it
if not os.path.exists(tmp_md):
shutil.copy(real_file,tmp_file)
"""
#give real header to the temp files using md as reference
overwrite=True
for contrast in ['dwi', 'b0', 'dwi_mask']:
tmp_file=os.path.join(work_dir,f'{subj}_tmp_{contrast}{ext}')
tmp2_file=os.path.join(work_dir,f'{subj}_tmp2_{contrast}{ext}')
final_file=os.path.join(work_dir,f'{subj}_{contrast}{ext}')
if ((not os.path.exists(tmp2_file) and not os.path.exists(final_file)) or overwrite):
if not os.path.exists(tmp_file):
raise Exception("Tmp file was not created, need to rerun previous processes")
else:
cmd = os.path.join(gunniespath, 'nifti_header_splicer.bash') + f' {reference_file} {tmp_file} {tmp2_file}'
os.system(cmd)
tmp_test_file = os.path.join(work_dir,f'{subj}_tmp2_test_{contrast}{ext}')
header_superpose(reference, tmp_file, outpath=tmp_test_file)
#write the relative orientation file here
if not os.path.isfile(orient_string) or overwrite:
if os.path.isfile(orient_string):
os.remove(orient_string)
file = os.path.join(work_dir,subj+'_tmp_dwi_mask'+ext);
cmd = 'bash ' + os.path.join(gunniespath,'find_relative_orientation_by_CoM.bash') + f' {reference_file} {file}'
orient_relative = subprocess.getoutput(cmd)
with open(orient_string, 'w') as f:
f.write(orient_relative)
else:
orient_relative = open(orient_string, mode='r').read()
#check extracted values from relative orientation vals
orientation_out = orient_relative.split(',')[0]
orientation_out = orientation_out.split(':')[1]
orientation_in = orient_relative.split(',')[1]
orientation_in = orientation_in.split(':')[1]
if verbose:
print(f'flexible orientation: {orientation_in}');
print(f'reference orientation: {orientation_out}');
#apply the orientation modification to specified contrasts
for contrast in ['dwi', 'b0', 'dwi_mask']:
img_in=os.path.join(work_dir,f'{subj}_tmp2_{contrast}{ext}')
img_out=os.path.join(work_dir,f'{subj}_{contrast}{ext}')
if not os.path.isfile(img_out) or overwrite:
if orientation_out != orientation_in:
print('TRYING TO REORIENT...b0 and dwi and mask')
if os.path.exists(img_in) and (not os.path.exists(img_out) or overwrite):
img_transform_exec(img_in, orientation_in, orientation_out, img_out, recenter=recenter)
if os.path.exists(img_out):
os.remove(img_in)
elif os.path.exists(img_out) and cleanup:
os.remove(img_in)
else:
shutil.move(img_in,img_out)
if bonusshortcutfolder is not None:
bonus_link = os.path.join(bonusshortcutfolder, f'{subj}_{contrast}{ext}')
if not os.path.exists(bonus_link) or overwrite:
buildlink(img_out, bonus_link)
mask = os.path.join(work_dir,f'{subj}_mask{ext}')
b0 = os.path.join(work_dir,f'{subj}_b0{ext}')
#if cleanup and os.path.exists(dwi_out) and os.path.exists(tmp_dwi_out):
# os.remove(tmp_dwi_out)
#refine mask (to check!!!!)
bonusmask=False
if bonusmask:
mask_new = mask.replace('_mask', '_new')
bet_cmd = f'bet {b0} {mask_new} -m -f 0.3'
os.system(bet_cmd)
mask_new = mask_new.replace('_new','_new_mask')
outpath=dwi_out.replace(".nii","_newmask.nii")
applymask_samespace(dwi_out, mask_new, outpath) #(add outpath if you want to save to different location rather than overwrite)
dwi_out=outpath
outpath=b0_out.replace(".nii","_newmask.nii")
applymask_samespace(b0_out, mask_new,outpath=outpath)
for contrast in ['fa0', 'rd', 'ad', 'md']:
real_file=largerfile(os.path.join(work_dir,f'*.fib.gz.{contrast}{ext}')) # It will be fun times if we ever have more than one match to this pattern...
if bonusmask:
real_file_new=real_file.replace("D.nii.gz","D_newmask.nii.gz")
applymask_samespace(real_file, mask_new,outpath=real_file_new)
real_file=real_file_new
contrast=contrast.replace('0','')
#linked_file=os.path.join(shortcut_dir,f'{subj}_{contrast}{ext}')
linked_file_w=os.path.join(work_dir,f'{subj}_{contrast}{ext}')
header_fix=True
if header_fix:
affine_superpose(dwi_out, real_file, transpose=transpose)
if not os.path.isfile(linked_file_w) or overwrite:
buildlink(real_file, linked_file_w)
if bonusshortcutfolder is not None:
blinked_file = os.path.join(bonusshortcutfolder, f'{subj}_{contrast}{ext}')
if not os.path.exists(blinked_file) or overwrite:
buildlink(real_file, blinked_file)
masking = 'bet'
denoise = 'mpca'
verbose = True
function_processes = np.int(max_processors / subject_processes)
results = []
if subject_processes > 1:
if function_processes > 1:
pool = MyPool(subject_processes)
else:
pool = mp.Pool(subject_processes)
results = pool.starmap_async(
launch_preprocessing,
[(proc_subjn + subject,
largerfile(
glob.glob(
os.path.join(os.path.join(
diffpath, "*" + subject + "*")))[0]), outpath, cleanup,
nominal_bval, SAMBA_inputs_folder, shortcuts_all_folder, gunniespath,
function_processes, masking, ref, transpose, overwrite, denoise,
recenter, verbose) for subject in subjects]).get()
else:
for subject in subjects:
max_size = 0
subjectpath = glob.glob(
os.path.join(os.path.join(outpath,
"diffusion*" + subject + "*")))[0]
print(subjectpath)
max_file = largerfile(subjectpath)
max_file = os.path.join(subjectpath,
"nii4D_" + proc_subjn + subject + ".nii.gz")
print(max_file)
results = pool.starmap_async(launch_preprocessing, [
launch_preprocessing(
proc_subjn + subject, max_file, outpath, cleanup, nominal_bval,
SAMBA_inputs_folder, shortcuts_all_folder, gunniespath,
function_processes, masking, ref, transpose, overwrite, denoise,
recenter, recenter, verbose) for subject in subjects
]).get()
else:
for subject in subjects:
max_size = 0
subjectpath = glob.glob(
os.path.join(os.path.join(diffpath,
"diffusion*" + subject + "*")))[0]
subject_outpath = os.path.join(
outpath, 'diffusion_prep_' + proc_subjn + subject)
max_file = largerfile(subjectpath)
if os.path.exists(
os.path.join(
subject_outpath,
f'{subject}_subjspace_fa.nii.gz')) and not overwrite:
print(f'already did subject {subject}')
elif os.path.exists(
os.path.join(
'/Volumes/Badea/Lab/APOE_symlink_pool/',
f'{subject}_subjspace_coreg.nii.gz')) and not overwrite:
print(
f'Could not find subject {subject} in main diffusion folder but result was found in SAMBA prep folder'
)
elif os.path.exists(
os.path.join(
'/Volumes/Data/Badea/Lab/mouse/VBM_20APOE01_chass_symmetric3_allAPOE-work/dwi/SyN_0p5_3_0p5_dwi/dwiMDT_NoNameYet_n32_i5/reg_images/',
pool = mp.Pool(subject_processes)
results = pool.starmap_async(launch_preprocessing, [
launch_preprocessing(
proc_subjn + subject, max_file, outpath, cleanup, nominal_bval,
SAMBA_inputs_folder, shortcuts_all_folder, gunniespath,
function_processes, masking, ref, transpose, overwrite, denoise,
recenter, recenter, verbose) for subject in subjects
]).get()
else:
for subject in subjects:
max_size = 0
subjectpath = glob.glob(
os.path.join(os.path.join(diffpath, subject + "*")))[0]
subject_outpath = os.path.join(
outpath, 'diffusion_prep_' + proc_subjn + subject)
max_file = largerfile(subjectpath, identifier=".nii")
if os.path.exists(
os.path.join(
subject_outpath,
f'{subject}_subjspace_fa.nii.gz')) and not overwrite:
print(f'already did subject {subject}')
elif os.path.exists(
os.path.join(
'/Volumes/Badea/Lab/APOE_symlink_pool/',
f'{subject}_subjspace_coreg.nii.gz')) and not overwrite:
print(
f'Could not find subject {subject} in main diffusion folder but result was found in SAMBA prep folder'
)
#elif os.path.exists(os.path.join(
# '/Volumes/Data/Badea/Lab/mouse/VBM_20APOE01_chass_symmetric3_allAPOE-work/dwi/SyN_0p5_3_0p5_dwi/dwiMDT_NoNameYet_n32_i5/reg_images/',
# f'{subject}_rd_to_MDT.nii.gz')) and not overwrite:
outpath=outpathsubj,
writeformat=writeformat,
fix=False,
overwrite=overwrite)
#fbvals, fbvecs = rewrite_subject_bvalues(dwipath, subject, outpath=outpath, writeformat=writeformat, overwrite=overwrite)
bvalfix = False
if bvalfix:
subject = "02432"
slicestocut = [6, 10, 14, 18, 20, 21, 23, 25, 27, 28, 30, 31, 35, 36]
bvaltofix = glob.glob(os.path.join(outpath, proc_name + subject,
"*bval*"))[0]
bvectofix = glob.glob(os.path.join(outpath, proc_name + subject,
"*bvec*"))[0]
outpathsubj = os.path.join(outpath, proc_name + subject)
dwifile = largerfile(outpathsubj)
bvals_cut = cut_bvals_bvecs(bvaltofix,
bvectofix,
slicestocut,
format="dsi")
os.remove(bvaltofix)
os.rename(bvals_cut, bvaltofix)
quickfix = False
if quickfix:
bval_file = "/Volumes/Data/Badea/ADdecode.01/Analysis/DWI/01912_bvals.txt"
bvec_file = "/Volumes/Data/Badea/ADdecode.01/Analysis/DWI/01912_bvec.txt"
bval_file, bvec_file = fix_bvals_bvecs(bval_file,
bvec_file,
outpath=outpath,
identifier="",
