def s_2_debug(params, inputs=[]): import time from subscripts.utilities import record_apptime, record_finish, write start_time = time.time() time.sleep(15) record_apptime(params, start_time, 2) record_finish(params)
def s2a_bedpostx(params, inputs=[]): import time from subscripts.utilities import run,smart_mkdir,smart_remove,write,record_start,record_apptime,record_finish,update_permissions,validate from os.path import exists,join,split from shutil import copyfile,rmtree sdir = params['sdir'] stdout = params['stdout'] container = params['container'] cores_per_task = params['cores_per_task'] use_gpu = params['use_gpu'] group = params['group'] record_start(params) start_time = time.time() bedpostx = join(sdir,"bedpostx_b1000") bedpostxResults = join(sdir,"bedpostx_b1000.bedpostX") th1 = join(bedpostxResults, "merged_th1samples") ph1 = join(bedpostxResults, "merged_ph1samples") th2 = join(bedpostxResults, "merged_th2samples") ph2 = join(bedpostxResults, "merged_ph2samples") dyads1 = join(bedpostxResults, "dyads1") dyads2 = join(bedpostxResults, "dyads2") brain_mask = join(bedpostxResults, "nodif_brain_mask") if exists(bedpostxResults): rmtree(bedpostxResults) smart_mkdir(bedpostx) smart_mkdir(bedpostxResults) copyfile(join(sdir,"data_eddy.nii.gz"),join(bedpostx,"data.nii.gz")) copyfile(join(sdir,"data_bet_mask.nii.gz"),join(bedpostx,"nodif_brain_mask.nii.gz")) copyfile(join(sdir,"bvals"),join(bedpostx,"bvals")) copyfile(join(sdir,"bvecs"),join(bedpostx,"bvecs")) if use_gpu: write(stdout, "Running Bedpostx with GPU") bedpostx_sh = join(sdir, "bedpostx.sh") smart_remove(bedpostx_sh) odir = split(sdir)[0] write(bedpostx_sh, "export CUDA_LIB_DIR=$CUDA_8_LIB_DIR\n" + "export LD_LIBRARY_PATH=$CUDA_LIB_DIR:$LD_LIBRARY_PATH") if container: write(bedpostx_sh, "bedpostx_gpu {} -NJOBS 4".format(bedpostx.replace(odir, "/share"))) else: write(bedpostx_sh, "bedpostx_gpu {} -NJOBS 4".format(bedpostx)) run("sh " + bedpostx_sh, params) # hacky validation step with open(stdout) as f: log_content = f.read() for i in range(1, 5): assert("{:d} parts processed out of 4".format(i) in log_content) else: write(stdout, "Running Bedpostx without GPU") run("bedpostx {}".format(bedpostx), params) run("make_dyadic_vectors {} {} {} {}".format(th1,ph1,brain_mask,dyads1), params) run("make_dyadic_vectors {} {} {} {}".format(th2,ph2,brain_mask,dyads2), params) validate(th1, params) validate(ph1, params) validate(dyads1, params) update_permissions(params) record_apptime(params, start_time, 1) record_finish(params)
def s1_4_dti_fit(params, inputs=[]): import time from subscripts.utilities import run, smart_remove, record_apptime, record_finish, update_permissions from os.path import join, exists from shutil import copyfile from glob import glob sdir = params['sdir'] stdout = params['stdout'] container = params['container'] cores_per_task = params['cores_per_task'] start_time = time.time() output_prefix = join(sdir, "data_eddy") output_data = join(sdir, "data_eddy.nii.gz") timeslices = glob("{}_tmp????.nii.gz".format(output_prefix)) timeslices.sort() bet = join(sdir, "data_bet.nii.gz") bvecs = join(sdir, "bvecs") bvals = join(sdir, "bvals") bet_mask = join(sdir, "data_bet_mask.nii.gz") dti_params = join(sdir, "DTIparams") dti_L1 = dti_params + "_L1.nii.gz" dti_L2 = dti_params + "_L2.nii.gz" dti_L3 = dti_params + "_L3.nii.gz" dti_MD = dti_params + "_MD.nii.gz" dti_RD = dti_params + "_RD.nii.gz" dti_MD = dti_params + "_MD.nii.gz" dti_AD = dti_params + "_AD.nii.gz" dti_FA = dti_params + "_FA.nii.gz" FA = join(sdir, "FA.nii.gz") run("fslmerge -t {} {}".format(output_data, " ".join(timeslices)), params) run("bet {} {} -m -f 0.3".format(output_data, bet), params) if exists(bet_mask): run( "dtifit --verbose -k {} -o {} -m {} -r {} -b {}".format( output_data, dti_params, bet_mask, bvecs, bvals), params) run( "fslmaths {} -add {} -add {} -div 3 {}".format( dti_L1, dti_L2, dti_L3, dti_MD), params) run("fslmaths {} -add {} -div 2 {}".format(dti_L2, dti_L3, dti_RD), params) copyfile(dti_L1, dti_AD) copyfile(dti_FA, FA) else: write(stdout, "Warning: failed to generate masked outputs") raise Exception( f"Failed BET step. Please check {stdout} for more info.") for i in glob("{}_tmp????.*".format(output_prefix)): smart_remove(i) for j in glob("{}_ref*".format(output_prefix)): smart_remove(j) update_permissions(params) record_apptime(params, start_time, 4) record_finish(params)
def s3_3_combine(params, volumes, inputs=[]): import time from subscripts.utilities import run,record_apptime,record_finish,update_permissions,write from os.path import join,exists sdir = params['sdir'] # volumes = params['volumes'] start_time = time.time() # run_vtk = join(sdir, 'run_vtk.py') outdir = join(sdir, 'fast_outdir') allvoxelscortsubcort = join(sdir,"allvoxelscortsubcort.nii.gz") total = join(outdir, 'FAtractsumsTwoway.nii.gz') run("fslmaths {} -mul 0 {}".format(allvoxelscortsubcort, total), params) for vol in volumes: vol_outdir = join(outdir, vol) pbtx_result = join(vol_outdir, 'fdt_paths.nii.gz') run("fslmaths {} -thrP 5 -bin {}".format(pbtx_result, pbtx_result), params) run("fslmaths {} -add {} {}".format(pbtx_result, total, total), params) waytotal = join(vol_outdir, "waytotal") # run('/opt/vtk/bin/vtkpython {} {} {} {}'.format(run_vtk, total, total + '.png', 256), params) update_permissions(params) record_apptime(params, start_time, 2) record_finish(params)
def s2b_2_process_vols(params, inputs=[]): import time from subscripts.utilities import run, smart_mkdir, smart_remove, write, record_apptime, record_finish, update_permissions from subscripts.maskseeds import maskseeds, saveallvoxels from os.path import exists, join, split, splitext from os import environ from shutil import copy from glob import glob sdir = params['sdir'] stdout = params['stdout'] container = params['container'] cores_per_task = params['cores_per_task'] group = params['group'] start_time = time.time() T1 = join(sdir, "T1.nii.gz") subject = split(sdir)[1] FA = join(sdir, "FA.nii.gz") aseg = join(sdir, "aseg.nii.gz") bs = join(sdir, "bs.nii.gz") FA2T1 = join(sdir, "FA2T1.mat") T12FA = join(sdir, "T12FA.mat") cort_label_dir = join(sdir, "label_cortical") cort_vol_dir = join(sdir, "volumes_cortical") cort_vol_dir_out = cort_vol_dir + "_s2fa" subcort_vol_dir = join(sdir, "volumes_subcortical") subcort_vol_dir_out = subcort_vol_dir + "_s2fa" terminationmask = join(sdir, "terminationmask.nii.gz") allvoxelscortsubcort = join(sdir, "allvoxelscortsubcort.nii.gz") intersection = join(sdir, "intersection.nii.gz") # exclusion_bsplusthalami = join(sdir,"exclusion_bsplusthalami.nii.gz") subcortical_index = [ '10:lh_thalamus', '11:lh_caudate', '12:lh_putamen', '13:lh_pallidum', '17:lh_hippocampus', '18:lh_amygdala', '26:lh_acumbens', '49:rh_thalamus', '50:rh_caudate', '51:rh_putamen', '52:rh_pallidum', '53:rh_hippocampus', '54:rh_amygdala', '58:rh_acumbens', ] EDI = join(sdir, "EDI") EDI_allvols = join(EDI, "allvols") smart_mkdir(cort_label_dir) smart_mkdir(cort_vol_dir) smart_mkdir(subcort_vol_dir) smart_mkdir(cort_vol_dir_out) smart_mkdir(subcort_vol_dir_out) smart_mkdir(EDI) smart_mkdir(EDI_allvols) if not container: environ['SUBJECTS_DIR'] = split(sdir)[0] run("mri_convert {} {} ".format(join(sdir, "mri", "brain.mgz"), T1), params) run("flirt -in {} -ref {} -omat {}".format(FA, T1, FA2T1), params) run("convert_xfm -omat {} -inverse {}".format(T12FA, FA2T1), params) run( "mri_annotation2label --subject {} --hemi rh --annotation aparc --outdir {}" .format(subject, cort_label_dir), params) run( "mri_annotation2label --subject {} --hemi lh --annotation aparc --outdir {}" .format(subject, cort_label_dir), params) for label in glob(join(cort_label_dir, "*.label")): vol_file = join(cort_vol_dir, splitext(split(label)[1])[0] + ".nii.gz") run( "mri_label2vol --label {} --temp {} --identity --o {}".format( label, T1, vol_file), params) run("mri_convert {} {}".format(join(sdir, "mri", "aseg.mgz"), aseg), params) for line in subcortical_index: num = line.split(":")[0].lstrip().rstrip() area = line.split(":")[1].lstrip().rstrip() area_out = join(subcort_vol_dir, area + ".nii.gz") write(stdout, "Processing " + area + ".nii.gz") run( "fslmaths {} -uthr {} -thr {} -bin {}".format( aseg, num, num, area_out), params) for volume in glob(join(cort_vol_dir, "*.nii.gz")): out_vol = join( cort_vol_dir_out, splitext(splitext(split(volume)[1])[0])[0] + "_s2fa.nii.gz") write( stdout, "Processing {} -> {}".format(split(volume)[1], split(out_vol)[1])) run( "flirt -in {} -ref {} -out {} -applyxfm -init {}".format( volume, FA, out_vol, T12FA), params) run("fslmaths {} -thr 0.2 -bin {} ".format(out_vol, out_vol), params) for volume in glob(join(subcort_vol_dir, "*.nii.gz")): out_vol = join( subcort_vol_dir_out, splitext(splitext(split(volume)[1])[0])[0] + "_s2fa.nii.gz") write( stdout, "Processing {} -> {}".format(split(volume)[1], split(out_vol)[1])) run( "flirt -in {} -ref {} -out {} -applyxfm -init {}".format( volume, FA, out_vol, T12FA), params) run("fslmaths {} -thr 0.2 -bin {}".format(out_vol, out_vol), params) run("fslmaths {} -mul 0 {}".format(FA, bs), params) # For now we fake a bs.nii.gz file maskseeds(sdir, join(cort_vol_dir + "_s2fa"), join(cort_vol_dir + "_s2fa_m"), 0.05, 1, 1, params) maskseeds(sdir, join(subcort_vol_dir + "_s2fa"), join(subcort_vol_dir + "_s2fa_m"), 0.05, 0.4, 0.4, params) saveallvoxels(sdir, join(cort_vol_dir + "_s2fa_m"), join(subcort_vol_dir + "_s2fa_m"), allvoxelscortsubcort, params) smart_remove(terminationmask) run("fslmaths {} -uthr .15 {}".format(FA, terminationmask), params) run("fslmaths {} -add {} {}".format(terminationmask, bs, terminationmask), params) run("fslmaths {} -bin {}".format(terminationmask, terminationmask), params) run( "fslmaths {} -mul {} {}".format(terminationmask, allvoxelscortsubcort, intersection), params) run( "fslmaths {} -sub {} {}".format(terminationmask, intersection, terminationmask), params) # run("fslmaths {} -add {} -add {} {}".format(bs, # join(subcort_vol_dir + "_s2fa_m","lh_thalamus_s2fa.nii.gz"), # join(subcort_vol_dir + "_s2fa_m","rh_thalamus_s2fa.nii.gz"), # exclusion_bsplusthalami), params) for file in glob(join(sdir, "volumes_cortical_s2fa", "*.nii.gz")): copy(file, EDI_allvols) for file in glob(join(sdir, "volumes_subcortical_s2fa", "*.nii.gz")): copy(file, EDI_allvols) update_permissions(params) record_apptime(params, start_time, 2) record_finish(params)
def s4_3_complete(params, inputs=[]): import time from subscripts.utilities import record_apptime, record_finish, update_permissions update_permissions(params) record_finish(params)
def s3_5_edi_combine(params, consensus_edges, inputs=[]): import time,tarfile from subscripts.utilities import run,smart_remove,smart_mkdir,write,record_apptime,record_finish, \ update_permissions,get_edges_from_file,strip_trailing_slash from os.path import join,exists,basename from shutil import copyfile pbtx_edge_list = params['pbtx_edge_list'] sdir = params['sdir'] stdout = params['stdout'] container = params['container'] start_time = time.time() pbtk_dir = join(sdir,"EDI","PBTKresults") connectome_dir = join(sdir,"EDI","CNTMresults") compress_pbtx_results = params['compress_pbtx_results'] consensus_dir = join(pbtk_dir,"twoway_consensus_edges") edi_maps = join(sdir,"EDI","EDImaps") edge_total = join(edi_maps,"FAtractsumsTwoway.nii.gz") tract_total = join(edi_maps,"FAtractsumsRaw.nii.gz") smart_remove(edi_maps) smart_mkdir(edi_maps) # Collect number of probtrackx tracts per voxel for edge in get_edges_from_file(pbtx_edge_list): a, b = edge a_to_b_formatted = "{}_s2fato{}_s2fa.nii.gz".format(a,b) a_to_b_file = join(pbtk_dir,a_to_b_formatted) if not exists(tract_total): copyfile(a_to_b_file, tract_total) else: run("fslmaths {0} -add {1} {1}".format(a_to_b_file, tract_total), params) # Collect number of parcel-to-parcel edges per voxel for edge in consensus_edges: a, b = edge consensus = join(consensus_dir, "{}_to_{}.nii.gz".format(a,b)) if not exists(consensus): write(stdout,"{} has been thresholded. See {} for details".format(edge, join(pbtk_dir, "zerosl.txt"))) continue if not exists(edge_total): copyfile(consensus, edge_total) else: run("fslmaths {0} -add {1} {1}".format(consensus, edge_total), params) if not exists(edge_total): write(stdout, "Error: Failed to generate {}".format(edge_total)) if compress_pbtx_results: pbtk_archive = strip_trailing_slash(pbtk_dir) + '.tar.gz' connectome_archive = strip_trailing_slash(connectome_dir) + '.tar.gz' write(stdout,"\nCompressing probtrackx output at {} and {}".format(pbtk_archive, connectome_archive)) smart_remove(pbtk_archive) smart_remove(connectome_archive) with tarfile.open(pbtk_archive, mode='w:gz') as archive: archive.add(pbtk_dir, recursive=True, arcname=basename(pbtk_dir)) with tarfile.open(connectome_archive, mode='w:gz') as archive: archive.add(connectome_dir, recursive=True, arcname=basename(connectome_dir)) smart_remove(pbtk_dir) smart_remove(connectome_dir) update_permissions(params) record_apptime(params, start_time, 4) record_finish(params)
def s3_3_combine(params, inputs=[]): import numpy as np import scipy.io import time from subscripts.utilities import record_apptime,record_finish,update_permissions,is_float,write,get_edges_from_file from os.path import join,exists from shutil import copyfile sdir = params['sdir'] stdout = params['stdout'] pbtx_sample_count = int(params['pbtx_sample_count']) pbtx_edge_list = params['pbtx_edge_list'] connectome_idx_list = params['connectome_idx_list'] connectome_idx_list_copy = join(sdir, 'connectome_idxs.txt') start_time = time.time() connectome_dir = join(sdir,"EDI","CNTMresults") oneway_list = join(sdir, "connectome_{}samples_oneway.txt".format(pbtx_sample_count)) twoway_list = join(sdir, "connectome_{}samples_twoway.txt".format(pbtx_sample_count)) oneway_nof = join(sdir, "connectome_{}samples_oneway_nof.mat".format(pbtx_sample_count)) # nof = number of fibers twoway_nof = join(sdir, "connectome_{}samples_twoway_nof.mat".format(pbtx_sample_count)) oneway_nof_normalized = join(sdir, "connectome_{}samples_oneway_nofn.mat".format(pbtx_sample_count)) # nofn = number of fibers, normalized twoway_nof_normalized = join(sdir, "connectome_{}samples_twoway_nofn.mat".format(pbtx_sample_count)) smart_remove(oneway_list) smart_remove(twoway_list) smart_remove(oneway_nof_normalized) smart_remove(twoway_nof_normalized) smart_remove(oneway_nof) smart_remove(twoway_nof) oneway_edges = {} twoway_edges = {} copyfile(connectome_idx_list, connectome_idx_list_copy) # give each subject a copy for reference vol_idxs = {} with open(connectome_idx_list) as f: lines = [x.strip() for x in f.readlines() if x] max_idx = -1 for line in lines: vol, idx = line.split(',', 1) idx = int(idx) vol_idxs[vol] = idx if idx > max_idx: max_idx = idx oneway_nof_normalized_matrix = np.zeros((max_idx+1, max_idx+1)) oneway_nof_matrix = np.zeros((max_idx+1, max_idx+1)) twoway_nof_normalized_matrix = np.zeros((max_idx+1, max_idx+1)) twoway_nof_matrix = np.zeros((max_idx+1, max_idx+1)) for edge in get_edges_from_file(pbtx_edge_list): a, b = edge edge_file = join(connectome_dir, "{}_to_{}.dot".format(a, b)) with open(edge_file) as f: chunks = [x.strip() for x in f.read().strip().split(' ') if x] a_to_b = (chunks[0], chunks[1]) b_to_a = (chunks[1], chunks[0]) waytotal_count = float(chunks[2]) fdt_count = float(chunks[3]) if b_to_a in twoway_edges: twoway_edges[b_to_a][0] += waytotal_count twoway_edges[b_to_a][1] += fdt_count else: twoway_edges[a_to_b] = [waytotal_count, fdt_count] oneway_edges[a_to_b] = [waytotal_count, fdt_count] for a_to_b in oneway_edges: a = a_to_b[0] b = a_to_b[1] for vol in a_to_b: if vol not in vol_idxs: write(stdout, 'Error: could not find {} in connectome idxs'.format(vol)) break else: write(oneway_list, "{} {} {} {}".format(a, b, oneway_edges[a_to_b][0], oneway_edges[a_to_b][1])) oneway_nof_matrix[vol_idxs[a]][vol_idxs[b]] = oneway_edges[a_to_b][0] oneway_nof_normalized_matrix[vol_idxs[a]][vol_idxs[b]] = oneway_edges[a_to_b][1] for a_to_b in twoway_edges: a = a_to_b[0] b = a_to_b[1] for vol in a_to_b: if vol not in vol_idxs: write(stdout, 'Error: could not find {} in connectome idxs'.format(vol)) break else: write(twoway_list, "{} {} {} {}".format(a, b, twoway_edges[a_to_b][0], twoway_edges[a_to_b][1])) twoway_nof_matrix[vol_idxs[a]][vol_idxs[b]] = twoway_edges[a_to_b][0] twoway_nof_normalized_matrix[vol_idxs[a]][vol_idxs[b]] = twoway_edges[a_to_b][1] scipy.io.savemat(oneway_nof, {'data': oneway_nof_matrix}) scipy.io.savemat(oneway_nof_normalized, {'data': oneway_nof_normalized_matrix}) scipy.io.savemat(twoway_nof, {'data': twoway_nof_matrix}) scipy.io.savemat(twoway_nof_normalized, {'data': twoway_nof_normalized_matrix}) update_permissions(params) record_apptime(params, start_time, 2) record_finish(params)