def s_2_debug(params, inputs=[]):
import time
from subscripts.utilities import record_apptime, record_finish, write
start_time = time.time()
time.sleep(15)
record_apptime(params, start_time, 2)
record_finish(params)
def s2a_bedpostx(params, inputs=[]):
import time
from subscripts.utilities import run,smart_mkdir,smart_remove,write,record_start,record_apptime,record_finish,update_permissions,validate
from os.path import exists,join,split
from shutil import copyfile,rmtree
sdir = params['sdir']
stdout = params['stdout']
container = params['container']
cores_per_task = params['cores_per_task']
use_gpu = params['use_gpu']
group = params['group']
record_start(params)
start_time = time.time()
bedpostx = join(sdir,"bedpostx_b1000")
bedpostxResults = join(sdir,"bedpostx_b1000.bedpostX")
th1 = join(bedpostxResults, "merged_th1samples")
ph1 = join(bedpostxResults, "merged_ph1samples")
th2 = join(bedpostxResults, "merged_th2samples")
ph2 = join(bedpostxResults, "merged_ph2samples")
dyads1 = join(bedpostxResults, "dyads1")
dyads2 = join(bedpostxResults, "dyads2")
brain_mask = join(bedpostxResults, "nodif_brain_mask")
if exists(bedpostxResults):
rmtree(bedpostxResults)
smart_mkdir(bedpostx)
smart_mkdir(bedpostxResults)
copyfile(join(sdir,"data_eddy.nii.gz"),join(bedpostx,"data.nii.gz"))
copyfile(join(sdir,"data_bet_mask.nii.gz"),join(bedpostx,"nodif_brain_mask.nii.gz"))
copyfile(join(sdir,"bvals"),join(bedpostx,"bvals"))
copyfile(join(sdir,"bvecs"),join(bedpostx,"bvecs"))
if use_gpu:
write(stdout, "Running Bedpostx with GPU")
bedpostx_sh = join(sdir, "bedpostx.sh")
smart_remove(bedpostx_sh)
odir = split(sdir)[0]
write(bedpostx_sh, "export CUDA_LIB_DIR=$CUDA_8_LIB_DIR\n" +
"export LD_LIBRARY_PATH=$CUDA_LIB_DIR:$LD_LIBRARY_PATH")
if container:
write(bedpostx_sh, "bedpostx_gpu {} -NJOBS 4".format(bedpostx.replace(odir, "/share")))
else:
write(bedpostx_sh, "bedpostx_gpu {} -NJOBS 4".format(bedpostx))
run("sh " + bedpostx_sh, params)
# hacky validation step
with open(stdout) as f:
log_content = f.read()
for i in range(1, 5):
assert("{:d} parts processed out of 4".format(i) in log_content)
else:
write(stdout, "Running Bedpostx without GPU")
run("bedpostx {}".format(bedpostx), params)
run("make_dyadic_vectors {} {} {} {}".format(th1,ph1,brain_mask,dyads1), params)
run("make_dyadic_vectors {} {} {} {}".format(th2,ph2,brain_mask,dyads2), params)
validate(th1, params)
validate(ph1, params)
validate(dyads1, params)
update_permissions(params)
record_apptime(params, start_time, 1)
record_finish(params)
def s1_4_dti_fit(params, inputs=[]):
import time
from subscripts.utilities import run, smart_remove, record_apptime, record_finish, update_permissions
from os.path import join, exists
from shutil import copyfile
from glob import glob
sdir = params['sdir']
stdout = params['stdout']
container = params['container']
cores_per_task = params['cores_per_task']
start_time = time.time()
output_prefix = join(sdir, "data_eddy")
output_data = join(sdir, "data_eddy.nii.gz")
timeslices = glob("{}_tmp????.nii.gz".format(output_prefix))
timeslices.sort()
bet = join(sdir, "data_bet.nii.gz")
bvecs = join(sdir, "bvecs")
bvals = join(sdir, "bvals")
bet_mask = join(sdir, "data_bet_mask.nii.gz")
dti_params = join(sdir, "DTIparams")
dti_L1 = dti_params + "_L1.nii.gz"
dti_L2 = dti_params + "_L2.nii.gz"
dti_L3 = dti_params + "_L3.nii.gz"
dti_MD = dti_params + "_MD.nii.gz"
dti_RD = dti_params + "_RD.nii.gz"
dti_MD = dti_params + "_MD.nii.gz"
dti_AD = dti_params + "_AD.nii.gz"
dti_FA = dti_params + "_FA.nii.gz"
FA = join(sdir, "FA.nii.gz")
run("fslmerge -t {} {}".format(output_data, " ".join(timeslices)), params)
run("bet {} {} -m -f 0.3".format(output_data, bet), params)
if exists(bet_mask):
run(
"dtifit --verbose -k {} -o {} -m {} -r {} -b {}".format(
output_data, dti_params, bet_mask, bvecs, bvals), params)
run(
"fslmaths {} -add {} -add {} -div 3 {}".format(
dti_L1, dti_L2, dti_L3, dti_MD), params)
run("fslmaths {} -add {} -div 2 {}".format(dti_L2, dti_L3, dti_RD),
params)
copyfile(dti_L1, dti_AD)
copyfile(dti_FA, FA)
else:
write(stdout, "Warning: failed to generate masked outputs")
raise Exception(
f"Failed BET step. Please check {stdout} for more info.")
for i in glob("{}_tmp????.*".format(output_prefix)):
smart_remove(i)
for j in glob("{}_ref*".format(output_prefix)):
smart_remove(j)
update_permissions(params)
record_apptime(params, start_time, 4)
record_finish(params)
def s3_3_combine(params, volumes, inputs=[]):
import time
from subscripts.utilities import run,record_apptime,record_finish,update_permissions,write
from os.path import join,exists
sdir = params['sdir']
# volumes = params['volumes']
start_time = time.time()
# run_vtk = join(sdir, 'run_vtk.py')
outdir = join(sdir, 'fast_outdir')
allvoxelscortsubcort = join(sdir,"allvoxelscortsubcort.nii.gz")
total = join(outdir, 'FAtractsumsTwoway.nii.gz')
run("fslmaths {} -mul 0 {}".format(allvoxelscortsubcort, total), params)
for vol in volumes:
vol_outdir = join(outdir, vol)
pbtx_result = join(vol_outdir, 'fdt_paths.nii.gz')
run("fslmaths {} -thrP 5 -bin {}".format(pbtx_result, pbtx_result), params)
run("fslmaths {} -add {} {}".format(pbtx_result, total, total), params)
waytotal = join(vol_outdir, "waytotal")
# run('/opt/vtk/bin/vtkpython {} {} {} {}'.format(run_vtk, total, total + '.png', 256), params)
update_permissions(params)
record_apptime(params, start_time, 2)
record_finish(params)
def s2b_2_process_vols(params, inputs=[]):
import time
from subscripts.utilities import run, smart_mkdir, smart_remove, write, record_apptime, record_finish, update_permissions
from subscripts.maskseeds import maskseeds, saveallvoxels
from os.path import exists, join, split, splitext
from os import environ
from shutil import copy
from glob import glob
sdir = params['sdir']
stdout = params['stdout']
container = params['container']
cores_per_task = params['cores_per_task']
group = params['group']
start_time = time.time()
T1 = join(sdir, "T1.nii.gz")
subject = split(sdir)[1]
FA = join(sdir, "FA.nii.gz")
aseg = join(sdir, "aseg.nii.gz")
bs = join(sdir, "bs.nii.gz")
FA2T1 = join(sdir, "FA2T1.mat")
T12FA = join(sdir, "T12FA.mat")
cort_label_dir = join(sdir, "label_cortical")
cort_vol_dir = join(sdir, "volumes_cortical")
cort_vol_dir_out = cort_vol_dir + "_s2fa"
subcort_vol_dir = join(sdir, "volumes_subcortical")
subcort_vol_dir_out = subcort_vol_dir + "_s2fa"
terminationmask = join(sdir, "terminationmask.nii.gz")
allvoxelscortsubcort = join(sdir, "allvoxelscortsubcort.nii.gz")
intersection = join(sdir, "intersection.nii.gz")
# exclusion_bsplusthalami = join(sdir,"exclusion_bsplusthalami.nii.gz")
subcortical_index = [
'10:lh_thalamus',
'11:lh_caudate',
'12:lh_putamen',
'13:lh_pallidum',
'17:lh_hippocampus',
'18:lh_amygdala',
'26:lh_acumbens',
'49:rh_thalamus',
'50:rh_caudate',
'51:rh_putamen',
'52:rh_pallidum',
'53:rh_hippocampus',
'54:rh_amygdala',
'58:rh_acumbens',
]
EDI = join(sdir, "EDI")
EDI_allvols = join(EDI, "allvols")
smart_mkdir(cort_label_dir)
smart_mkdir(cort_vol_dir)
smart_mkdir(subcort_vol_dir)
smart_mkdir(cort_vol_dir_out)
smart_mkdir(subcort_vol_dir_out)
smart_mkdir(EDI)
smart_mkdir(EDI_allvols)
if not container:
environ['SUBJECTS_DIR'] = split(sdir)[0]
run("mri_convert {} {} ".format(join(sdir, "mri", "brain.mgz"), T1),
params)
run("flirt -in {} -ref {} -omat {}".format(FA, T1, FA2T1), params)
run("convert_xfm -omat {} -inverse {}".format(T12FA, FA2T1), params)
run(
"mri_annotation2label --subject {} --hemi rh --annotation aparc --outdir {}"
.format(subject, cort_label_dir), params)
run(
"mri_annotation2label --subject {} --hemi lh --annotation aparc --outdir {}"
.format(subject, cort_label_dir), params)
for label in glob(join(cort_label_dir, "*.label")):
vol_file = join(cort_vol_dir, splitext(split(label)[1])[0] + ".nii.gz")
run(
"mri_label2vol --label {} --temp {} --identity --o {}".format(
label, T1, vol_file), params)
run("mri_convert {} {}".format(join(sdir, "mri", "aseg.mgz"), aseg),
params)
for line in subcortical_index:
num = line.split(":")[0].lstrip().rstrip()
area = line.split(":")[1].lstrip().rstrip()
area_out = join(subcort_vol_dir, area + ".nii.gz")
write(stdout, "Processing " + area + ".nii.gz")
run(
"fslmaths {} -uthr {} -thr {} -bin {}".format(
aseg, num, num, area_out), params)
for volume in glob(join(cort_vol_dir, "*.nii.gz")):
out_vol = join(
cort_vol_dir_out,
splitext(splitext(split(volume)[1])[0])[0] + "_s2fa.nii.gz")
write(
stdout,
"Processing {} -> {}".format(split(volume)[1],
split(out_vol)[1]))
run(
"flirt -in {} -ref {} -out {} -applyxfm -init {}".format(
volume, FA, out_vol, T12FA), params)
run("fslmaths {} -thr 0.2 -bin {} ".format(out_vol, out_vol), params)
for volume in glob(join(subcort_vol_dir, "*.nii.gz")):
out_vol = join(
subcort_vol_dir_out,
splitext(splitext(split(volume)[1])[0])[0] + "_s2fa.nii.gz")
write(
stdout,
"Processing {} -> {}".format(split(volume)[1],
split(out_vol)[1]))
run(
"flirt -in {} -ref {} -out {} -applyxfm -init {}".format(
volume, FA, out_vol, T12FA), params)
run("fslmaths {} -thr 0.2 -bin {}".format(out_vol, out_vol), params)
run("fslmaths {} -mul 0 {}".format(FA, bs),
params) # For now we fake a bs.nii.gz file
maskseeds(sdir, join(cort_vol_dir + "_s2fa"),
join(cort_vol_dir + "_s2fa_m"), 0.05, 1, 1, params)
maskseeds(sdir, join(subcort_vol_dir + "_s2fa"),
join(subcort_vol_dir + "_s2fa_m"), 0.05, 0.4, 0.4, params)
saveallvoxels(sdir, join(cort_vol_dir + "_s2fa_m"),
join(subcort_vol_dir + "_s2fa_m"), allvoxelscortsubcort,
params)
smart_remove(terminationmask)
run("fslmaths {} -uthr .15 {}".format(FA, terminationmask), params)
run("fslmaths {} -add {} {}".format(terminationmask, bs, terminationmask),
params)
run("fslmaths {} -bin {}".format(terminationmask, terminationmask), params)
run(
"fslmaths {} -mul {} {}".format(terminationmask, allvoxelscortsubcort,
intersection), params)
run(
"fslmaths {} -sub {} {}".format(terminationmask, intersection,
terminationmask), params)
# run("fslmaths {} -add {} -add {} {}".format(bs,
# join(subcort_vol_dir + "_s2fa_m","lh_thalamus_s2fa.nii.gz"),
# join(subcort_vol_dir + "_s2fa_m","rh_thalamus_s2fa.nii.gz"),
# exclusion_bsplusthalami), params)
for file in glob(join(sdir, "volumes_cortical_s2fa", "*.nii.gz")):
copy(file, EDI_allvols)
for file in glob(join(sdir, "volumes_subcortical_s2fa", "*.nii.gz")):
copy(file, EDI_allvols)
update_permissions(params)
record_apptime(params, start_time, 2)
record_finish(params)
def s4_3_complete(params, inputs=[]):
import time
from subscripts.utilities import record_apptime, record_finish, update_permissions
update_permissions(params)
record_finish(params)
def s3_5_edi_combine(params, consensus_edges, inputs=[]):
import time,tarfile
from subscripts.utilities import run,smart_remove,smart_mkdir,write,record_apptime,record_finish, \
update_permissions,get_edges_from_file,strip_trailing_slash
from os.path import join,exists,basename
from shutil import copyfile
pbtx_edge_list = params['pbtx_edge_list']
sdir = params['sdir']
stdout = params['stdout']
container = params['container']
start_time = time.time()
pbtk_dir = join(sdir,"EDI","PBTKresults")
connectome_dir = join(sdir,"EDI","CNTMresults")
compress_pbtx_results = params['compress_pbtx_results']
consensus_dir = join(pbtk_dir,"twoway_consensus_edges")
edi_maps = join(sdir,"EDI","EDImaps")
edge_total = join(edi_maps,"FAtractsumsTwoway.nii.gz")
tract_total = join(edi_maps,"FAtractsumsRaw.nii.gz")
smart_remove(edi_maps)
smart_mkdir(edi_maps)
# Collect number of probtrackx tracts per voxel
for edge in get_edges_from_file(pbtx_edge_list):
a, b = edge
a_to_b_formatted = "{}_s2fato{}_s2fa.nii.gz".format(a,b)
a_to_b_file = join(pbtk_dir,a_to_b_formatted)
if not exists(tract_total):
copyfile(a_to_b_file, tract_total)
else:
run("fslmaths {0} -add {1} {1}".format(a_to_b_file, tract_total), params)
# Collect number of parcel-to-parcel edges per voxel
for edge in consensus_edges:
a, b = edge
consensus = join(consensus_dir, "{}_to_{}.nii.gz".format(a,b))
if not exists(consensus):
write(stdout,"{} has been thresholded. See {} for details".format(edge, join(pbtk_dir, "zerosl.txt")))
continue
if not exists(edge_total):
copyfile(consensus, edge_total)
else:
run("fslmaths {0} -add {1} {1}".format(consensus, edge_total), params)
if not exists(edge_total):
write(stdout, "Error: Failed to generate {}".format(edge_total))
if compress_pbtx_results:
pbtk_archive = strip_trailing_slash(pbtk_dir) + '.tar.gz'
connectome_archive = strip_trailing_slash(connectome_dir) + '.tar.gz'
write(stdout,"\nCompressing probtrackx output at {} and {}".format(pbtk_archive, connectome_archive))
smart_remove(pbtk_archive)
smart_remove(connectome_archive)
with tarfile.open(pbtk_archive, mode='w:gz') as archive:
archive.add(pbtk_dir, recursive=True, arcname=basename(pbtk_dir))
with tarfile.open(connectome_archive, mode='w:gz') as archive:
archive.add(connectome_dir, recursive=True, arcname=basename(connectome_dir))
smart_remove(pbtk_dir)
smart_remove(connectome_dir)
update_permissions(params)
record_apptime(params, start_time, 4)
record_finish(params)
def s3_3_combine(params, inputs=[]):
import numpy as np
import scipy.io
import time
from subscripts.utilities import record_apptime,record_finish,update_permissions,is_float,write,get_edges_from_file
from os.path import join,exists
from shutil import copyfile
sdir = params['sdir']
stdout = params['stdout']
pbtx_sample_count = int(params['pbtx_sample_count'])
pbtx_edge_list = params['pbtx_edge_list']
connectome_idx_list = params['connectome_idx_list']
connectome_idx_list_copy = join(sdir, 'connectome_idxs.txt')
start_time = time.time()
connectome_dir = join(sdir,"EDI","CNTMresults")
oneway_list = join(sdir, "connectome_{}samples_oneway.txt".format(pbtx_sample_count))
twoway_list = join(sdir, "connectome_{}samples_twoway.txt".format(pbtx_sample_count))
oneway_nof = join(sdir, "connectome_{}samples_oneway_nof.mat".format(pbtx_sample_count)) # nof = number of fibers
twoway_nof = join(sdir, "connectome_{}samples_twoway_nof.mat".format(pbtx_sample_count))
oneway_nof_normalized = join(sdir, "connectome_{}samples_oneway_nofn.mat".format(pbtx_sample_count)) # nofn = number of fibers, normalized
twoway_nof_normalized = join(sdir, "connectome_{}samples_twoway_nofn.mat".format(pbtx_sample_count))
smart_remove(oneway_list)
smart_remove(twoway_list)
smart_remove(oneway_nof_normalized)
smart_remove(twoway_nof_normalized)
smart_remove(oneway_nof)
smart_remove(twoway_nof)
oneway_edges = {}
twoway_edges = {}
copyfile(connectome_idx_list, connectome_idx_list_copy) # give each subject a copy for reference
vol_idxs = {}
with open(connectome_idx_list) as f:
lines = [x.strip() for x in f.readlines() if x]
max_idx = -1
for line in lines:
vol, idx = line.split(',', 1)
idx = int(idx)
vol_idxs[vol] = idx
if idx > max_idx:
max_idx = idx
oneway_nof_normalized_matrix = np.zeros((max_idx+1, max_idx+1))
oneway_nof_matrix = np.zeros((max_idx+1, max_idx+1))
twoway_nof_normalized_matrix = np.zeros((max_idx+1, max_idx+1))
twoway_nof_matrix = np.zeros((max_idx+1, max_idx+1))
for edge in get_edges_from_file(pbtx_edge_list):
a, b = edge
edge_file = join(connectome_dir, "{}_to_{}.dot".format(a, b))
with open(edge_file) as f:
chunks = [x.strip() for x in f.read().strip().split(' ') if x]
a_to_b = (chunks[0], chunks[1])
b_to_a = (chunks[1], chunks[0])
waytotal_count = float(chunks[2])
fdt_count = float(chunks[3])
if b_to_a in twoway_edges:
twoway_edges[b_to_a][0] += waytotal_count
twoway_edges[b_to_a][1] += fdt_count
else:
twoway_edges[a_to_b] = [waytotal_count, fdt_count]
oneway_edges[a_to_b] = [waytotal_count, fdt_count]
for a_to_b in oneway_edges:
a = a_to_b[0]
b = a_to_b[1]
for vol in a_to_b:
if vol not in vol_idxs:
write(stdout, 'Error: could not find {} in connectome idxs'.format(vol))
break
else:
write(oneway_list, "{} {} {} {}".format(a, b, oneway_edges[a_to_b][0], oneway_edges[a_to_b][1]))
oneway_nof_matrix[vol_idxs[a]][vol_idxs[b]] = oneway_edges[a_to_b][0]
oneway_nof_normalized_matrix[vol_idxs[a]][vol_idxs[b]] = oneway_edges[a_to_b][1]
for a_to_b in twoway_edges:
a = a_to_b[0]
b = a_to_b[1]
for vol in a_to_b:
if vol not in vol_idxs:
write(stdout, 'Error: could not find {} in connectome idxs'.format(vol))
break
else:
write(twoway_list, "{} {} {} {}".format(a, b, twoway_edges[a_to_b][0], twoway_edges[a_to_b][1]))
twoway_nof_matrix[vol_idxs[a]][vol_idxs[b]] = twoway_edges[a_to_b][0]
twoway_nof_normalized_matrix[vol_idxs[a]][vol_idxs[b]] = twoway_edges[a_to_b][1]
scipy.io.savemat(oneway_nof, {'data': oneway_nof_matrix})
scipy.io.savemat(oneway_nof_normalized, {'data': oneway_nof_normalized_matrix})
scipy.io.savemat(twoway_nof, {'data': twoway_nof_matrix})
scipy.io.savemat(twoway_nof_normalized, {'data': twoway_nof_normalized_matrix})
update_permissions(params)
record_apptime(params, start_time, 2)
record_finish(params)
