def test_run_many_subs(): import numpy as np from qc import cpac_many_subs session_id = 1 pipeline_dir = '/data/Projects/ABIDE_Initiative/CPAC/Output_2013-11-22/pipeline_MerrittIsland' sublist_file = "/data/Projects/ABIDE_Initiative/scripts/for_grant/10_raw_anat/z_sublist.txt" sublist = np.loadtxt(sublist_file) sublist = [ "%07i" % s for s in sublist ] df1 = cpac_many_subs(sublist[:3], pipeline_dir, session_id, ncores=3) df2 = cpac_many_subs(sublist[:3], pipeline_dir, session_id, ncores=1)
def test_paths(): from qc import CpacPaths, CpacFunctionalPaths # subjects without a functional scan 1 bad_subs = ["0050155", "0050165"] sublist = [ s for s in sublist if s not in bad_subs ] for subject_id in sublist: print subject_id cpac_paths = CpacFunctionalPaths(pipeline_dir, subject_id, session_id, 1) func_path = cpac_paths.extract("motion_correct") # 0050142 (was an issue but now is fine) subject_id = "0050142" cpac_paths = CpacFunctionalPaths(pipeline_dir, subject_id, session_id, 1) paths = cpac_paths.extract_all("motion_correct") path = cpac_paths.extract("motion_correct") # 0050155 (list index out of range) subject_id = "0050155" cpac_paths = CpacFunctionalPaths(pipeline_dir, subject_id, session_id, 1) paths = cpac_paths.extract_all("motion_correct") path = cpac_paths.extract("motion_correct") # fixed now bad_subs = ["0050155", "0050165"] (bad_subs, df) = cpac_many_subs("temporal", bad_subs, pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=1)
pipeline_dir = "/data/Projects/CoRR/preproc/output/pipeline_corr_qc_preproc" for session_id in session_ids: print "=====" print "SESSION: %i" % session_id print "time: %s" % strftime("%Y-%m-%d %H:%M:%S") # Subjects sublist_file = "%s/subject_lists/session%02i_scan%02i.txt" % (outdir, session_id, scan_id) sublist = np.loadtxt(sublist_file, dtype=str).tolist() # COMPUTE COMPUTE COMPUTE ... start = timeit.default_timer() #(bad_subs, df) = cpac_many_subs("spatial", sublist[:10], pipeline_dir, session_id=session_id, ncores=1) #(bad_subs, df) = cpac_many_subs("spatial", ["0026048"], pipeline_dir, session_id=session_id, ncores=1) (bad_subs, df) = cpac_many_subs("spatial", sublist, pipeline_dir, session_id=session_id, ncores=ncores) stop = timeit.default_timer() duration = (stop-start)/60. # in minutes print "time elapsed (mins):", duration # Save Output print "...saving output" ## df if not os.path.exists(outdir): os.mkdir(outdir) outfile = os.path.join(outdir, "qc_spatial_session%02i.csv" % session_id) df.to_csv(outfile) ## bad subjects outfile = os.path.join(outdir, "qc_spatial_bad_subs_session%02i.txt" % session_id) np.savetxt(outfile, bad_subs, fmt="%s")
print "Rerunning %i subjects out of %i" % (len(new_sublist), len(sublist)) sublist = new_sublist out = out.ix[~np.isnan(out.efc),:] else: print "Output already exists, done" raise SystemExit(1) else: out = None # COMPUTE COMPUTE COMPUTE ... #(bad_subs, df) = cpac_many_subs("temporal", sublist[:2], pipeline_dir, # session_id=session_id, scan_id=scan_id, ncores=1, # pipeline_dir2=pipeline_dir2, verbose=True, # motion_threshold=1.5, opts=["fd"]) # dvars, fd, outlier, quality (bad_subs, df) = cpac_many_subs("temporal", sublist, pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=ncores, pipeline_dir2=pipeline_dir2, motion_threshold=0.2, opts=["fd"]) # dvars, fd, outlier, quality) # Combine with previous if needed if out is not None and df.shape[0] != 0: print "...combining old with new" # concatenate by rows out = out[[ c for c in out.columns if c.find("Unnamed") == -1 ]] df = out.append(df) # Save Output print "...saving output" ## df if not os.path.exists(outdir): os.mkdir(outdir) outfile = os.path.join(outdir, "qc_temporal_fd.csv")
# sublist = [sublist] #print "Rerunning %i subjects" % len(sublist) #continue else: print "Output already exists, skipping" continue # exclude 0025770 #sublist = [ s for s in sublist if s != "0025770" ] # COMPUTE COMPUTE COMPUTE ... start = timeit.default_timer() #(bad_subs, df) = cpac_many_subs("temporal", sublist[:2], pipeline_dir, # session_id=session_id, scan_id=scan_id, ncores=1) (bad_subs, df) = cpac_many_subs("temporal", sublist, pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=ncores) stop = timeit.default_timer() duration = (stop-start)/60. # in minutes print "time elapsed (mins):", duration # Save Output print "...saving output" ## df if not os.path.exists(outdir): os.mkdir(outdir) outfile = os.path.join(outdir, "qc_temporal_epi_session%02i_scan%02i.csv" % (session_id, scan_id)) df.to_csv(outfile) ## bad subjects outfile = os.path.join(outdir, "qc_temporal_epi_bad_subs_session%02i_scan%02i.txt" % (session_id, scan_id)) np.savetxt(outfile, bad_subs, fmt="%s")
elif np.isnan(out.efc).any(): print "Some of the EFC are missing, rerunning those" new_sublist = out.subject[np.isnan(out.efc)].tolist() new_sublist = [ "%07i" % int(s) for s in new_sublist ] print "Rerunning %i subjects out of %i" % (len(new_sublist), len(sublist)) sublist = new_sublist out = out.ix[~np.isnan(out.efc),:] else: print "Output already exists, done" raise SystemExit(1) else: out = None # COMPUTE COMPUTE COMPUTE ... #(bad_subs, df) = cpac_many_subs("spatial_epi", sublist[:2], pipeline_dir, session_id=session_id, ncores=1) (bad_subs, df) = cpac_many_subs("spatial_epi", sublist, pipeline_dir, session_id=session_id, ncores=ncores, out_vox=True) # Combine with previous if needed if out is not None: print "...combining old with new" # concatenate by rows out = out[[ c for c in out.columns if c.find("Unnamed") == -1 ]] if df.shape[0] == 0: df = out else: df = out.append(df) # Save Output print "...saving output" ## df if not os.path.exists(outdir):
print "Some of the EFC are missing, rerunning those" new_sublist = out.subject[np.isnan(out.efc)].tolist() new_sublist = [ "%07i" % int(s) for s in new_sublist ] print "Rerunning %i subjects out of %i" % (len(new_sublist), len(sublist)) sublist = new_sublist out = out.ix[~np.isnan(out.efc),:] else: print "Output already exists, done" raise SystemExit(1) else: out = None # COMPUTE COMPUTE COMPUTE ... #(bad_subs, df) = cpac_many_subs("temporal", sublist[:2], pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=1, # pipeline_dir2=pipeline_dir2, verbose=True, motion_threshold=1.5) (bad_subs, df) = cpac_many_subs("temporal", sublist, pipeline_dir, session_id=session_id, scan_id=scan_id, ncores=ncores, pipeline_dir2=pipeline_dir2, motion_threshold=0.2) # Combine with previous if needed if out is not None and df.shape[0] != 0: print "...combining old with new" # concatenate by rows out = out[[ c for c in out.columns if c.find("Unnamed") == -1 ]] df = out.append(df) # Save Output print "...saving output" ## df if not os.path.exists(outdir): os.mkdir(outdir) outfile = os.path.join(outdir, "qc_temporal.csv") df.to_csv(outfile)