def test_run_many_subs():
import numpy as np
from qc import cpac_many_subs
session_id = 1
pipeline_dir = '/data/Projects/ABIDE_Initiative/CPAC/Output_2013-11-22/pipeline_MerrittIsland'
sublist_file = "/data/Projects/ABIDE_Initiative/scripts/for_grant/10_raw_anat/z_sublist.txt"
sublist = np.loadtxt(sublist_file)
sublist = [ "%07i" % s for s in sublist ]
df1 = cpac_many_subs(sublist[:3], pipeline_dir, session_id, ncores=3)
df2 = cpac_many_subs(sublist[:3], pipeline_dir, session_id, ncores=1)
def test_paths():
from qc import CpacPaths, CpacFunctionalPaths
# subjects without a functional scan 1
bad_subs = ["0050155", "0050165"]
sublist = [ s for s in sublist if s not in bad_subs ]
for subject_id in sublist:
print subject_id
cpac_paths = CpacFunctionalPaths(pipeline_dir, subject_id, session_id, 1)
func_path = cpac_paths.extract("motion_correct")
# 0050142 (was an issue but now is fine)
subject_id = "0050142"
cpac_paths = CpacFunctionalPaths(pipeline_dir, subject_id, session_id, 1)
paths = cpac_paths.extract_all("motion_correct")
path = cpac_paths.extract("motion_correct")
# 0050155 (list index out of range)
subject_id = "0050155"
cpac_paths = CpacFunctionalPaths(pipeline_dir, subject_id, session_id, 1)
paths = cpac_paths.extract_all("motion_correct")
path = cpac_paths.extract("motion_correct")
# fixed now
bad_subs = ["0050155", "0050165"]
(bad_subs, df) = cpac_many_subs("temporal", bad_subs, pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=1)
pipeline_dir = "/data/Projects/CoRR/preproc/output/pipeline_corr_qc_preproc"
for session_id in session_ids:
print "====="
print "SESSION: %i" % session_id
print "time: %s" % strftime("%Y-%m-%d %H:%M:%S")
# Subjects
sublist_file = "%s/subject_lists/session%02i_scan%02i.txt" % (outdir, session_id, scan_id)
sublist = np.loadtxt(sublist_file, dtype=str).tolist()
# COMPUTE COMPUTE COMPUTE ...
start = timeit.default_timer()
#(bad_subs, df) = cpac_many_subs("spatial", sublist[:10], pipeline_dir, session_id=session_id, ncores=1)
#(bad_subs, df) = cpac_many_subs("spatial", ["0026048"], pipeline_dir, session_id=session_id, ncores=1)
(bad_subs, df) = cpac_many_subs("spatial", sublist, pipeline_dir, session_id=session_id, ncores=ncores)
stop = timeit.default_timer()
duration = (stop-start)/60. # in minutes
print "time elapsed (mins):", duration
# Save Output
print "...saving output"
## df
if not os.path.exists(outdir):
os.mkdir(outdir)
outfile = os.path.join(outdir, "qc_spatial_session%02i.csv" % session_id)
df.to_csv(outfile)
## bad subjects
outfile = os.path.join(outdir, "qc_spatial_bad_subs_session%02i.txt" % session_id)
np.savetxt(outfile, bad_subs, fmt="%s")
print "Rerunning %i subjects out of %i" % (len(new_sublist), len(sublist))
sublist = new_sublist
out = out.ix[~np.isnan(out.efc),:]
else:
print "Output already exists, done"
raise SystemExit(1)
else:
out = None
# COMPUTE COMPUTE COMPUTE ...
#(bad_subs, df) = cpac_many_subs("temporal", sublist[:2], pipeline_dir,
# session_id=session_id, scan_id=scan_id, ncores=1,
# pipeline_dir2=pipeline_dir2, verbose=True,
# motion_threshold=1.5, opts=["fd"]) # dvars, fd, outlier, quality
(bad_subs, df) = cpac_many_subs("temporal", sublist, pipeline_dir,
session_id=session_id, scan_id=scan_id, ncores=ncores,
pipeline_dir2=pipeline_dir2, motion_threshold=0.2,
opts=["fd"]) # dvars, fd, outlier, quality)
# Combine with previous if needed
if out is not None and df.shape[0] != 0:
print "...combining old with new"
# concatenate by rows
out = out[[ c for c in out.columns if c.find("Unnamed") == -1 ]]
df = out.append(df)
# Save Output
print "...saving output"
## df
if not os.path.exists(outdir):
os.mkdir(outdir)
outfile = os.path.join(outdir, "qc_temporal_fd.csv")
# sublist = [sublist]
#print "Rerunning %i subjects" % len(sublist)
#continue
else:
print "Output already exists, skipping"
continue
# exclude 0025770
#sublist = [ s for s in sublist if s != "0025770" ]
# COMPUTE COMPUTE COMPUTE ...
start = timeit.default_timer()
#(bad_subs, df) = cpac_many_subs("temporal", sublist[:2], pipeline_dir,
# session_id=session_id, scan_id=scan_id, ncores=1)
(bad_subs, df) = cpac_many_subs("temporal", sublist, pipeline_dir,
session_id=session_id, scan_id=scan_id, ncores=ncores)
stop = timeit.default_timer()
duration = (stop-start)/60. # in minutes
print "time elapsed (mins):", duration
# Save Output
print "...saving output"
## df
if not os.path.exists(outdir):
os.mkdir(outdir)
outfile = os.path.join(outdir, "qc_temporal_epi_session%02i_scan%02i.csv" % (session_id, scan_id))
df.to_csv(outfile)
## bad subjects
outfile = os.path.join(outdir, "qc_temporal_epi_bad_subs_session%02i_scan%02i.txt" % (session_id, scan_id))
np.savetxt(outfile, bad_subs, fmt="%s")
elif np.isnan(out.efc).any():
print "Some of the EFC are missing, rerunning those"
new_sublist = out.subject[np.isnan(out.efc)].tolist()
new_sublist = [ "%07i" % int(s) for s in new_sublist ]
print "Rerunning %i subjects out of %i" % (len(new_sublist), len(sublist))
sublist = new_sublist
out = out.ix[~np.isnan(out.efc),:]
else:
print "Output already exists, done"
raise SystemExit(1)
else:
out = None
# COMPUTE COMPUTE COMPUTE ...
#(bad_subs, df) = cpac_many_subs("spatial_epi", sublist[:2], pipeline_dir, session_id=session_id, ncores=1)
(bad_subs, df) = cpac_many_subs("spatial_epi", sublist, pipeline_dir, session_id=session_id, ncores=ncores, out_vox=True)
# Combine with previous if needed
if out is not None:
print "...combining old with new"
# concatenate by rows
out = out[[ c for c in out.columns if c.find("Unnamed") == -1 ]]
if df.shape[0] == 0:
df = out
else:
df = out.append(df)
# Save Output
print "...saving output"
## df
if not os.path.exists(outdir):
print "Some of the EFC are missing, rerunning those"
new_sublist = out.subject[np.isnan(out.efc)].tolist()
new_sublist = [ "%07i" % int(s) for s in new_sublist ]
print "Rerunning %i subjects out of %i" % (len(new_sublist), len(sublist))
sublist = new_sublist
out = out.ix[~np.isnan(out.efc),:]
else:
print "Output already exists, done"
raise SystemExit(1)
else:
out = None
# COMPUTE COMPUTE COMPUTE ...
#(bad_subs, df) = cpac_many_subs("temporal", sublist[:2], pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=1,
# pipeline_dir2=pipeline_dir2, verbose=True, motion_threshold=1.5)
(bad_subs, df) = cpac_many_subs("temporal", sublist, pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=ncores, pipeline_dir2=pipeline_dir2, motion_threshold=0.2)
# Combine with previous if needed
if out is not None and df.shape[0] != 0:
print "...combining old with new"
# concatenate by rows
out = out[[ c for c in out.columns if c.find("Unnamed") == -1 ]]
df = out.append(df)
# Save Output
print "...saving output"
## df
if not os.path.exists(outdir):
os.mkdir(outdir)
outfile = os.path.join(outdir, "qc_temporal.csv")
df.to_csv(outfile)
