def export_roi(h5file, conds, rois, out_dir): E = Exp(h5file) # select conditions and rois (all if none are specified) conds = [v for k, v in E.f['conds'].iteritems() if not conds or k in conds] rois = [v for k, v in E.f['rois'].iteritems() if not rois or k in rois] for cond in conds: cond_name = cond.name.split('/')[-1].encode() print cond_name for roi in rois: roi_name = roi.name.split('/')[-1].encode() print '\t', roi_name roi = roi[...] sub_tc = { run.grp.parent.name.split('/')[1]: run.load(roi=roi, threshold=True, standardized=True) for run in E.iter_runs(cond) } data = pd.DataFrame(sub_tc) data['roi'] = roi_name data['cond'] = cond_name outname = os.path.join(out_dir or '', cond_name, roi_name + '.csv') mkdir_p(os.path.dirname(outname)) data.to_csv(outname)
def export_roi(h5file, conds, rois, out_dir): E = Exp(h5file) # select conditions and rois (all if none are specified) conds = [v for k, v in E.f['conds'].iteritems() if not conds or k in conds] rois = [v for k, v in E.f['rois'].iteritems() if not rois or k in rois] for cond in conds: cond_name= cond.name.split('/')[-1].encode() print cond_name for roi in rois: roi_name = roi.name.split('/')[-1].encode() print '\t', roi_name roi = roi[...] sub_tc = {run.grp.parent.name.split('/')[1] : run.load(roi=roi, threshold=True, standardized=True) for run in E.iter_runs(cond)} data = pd.DataFrame(sub_tc) data['roi'] = roi_name data['cond'] = cond_name outname = os.path.join(out_dir or '', cond_name, roi_name + '.csv') mkdir_p(os.path.dirname(outname)) data.to_csv(outname)
def permute_isc_within(a, b, x, outfile, mask='', isc_only=False, hdf5=None, thresh=6000, n_pass=.7, n_reps=1000, t=False): import numpy as np from pycorr.funcs_correlate import crosscor, intersubcorr from pycorr.statistics import perm, isc_corrmat_within_diff from pycorr.subject import Run, Exp from pycorr.pietools import mkdir_p, parse_section, arr_slice # TASK ID so script knows where to slice, converts SGE_TASK_ID to 0-indexed ID = parse_section(x) if x is not None else int(os.environ['SGE_TASK_ID']) - 1 # OUTPUTS out = {} # Load and Slice data --------------------------------------------------------- mask = np.load(mask) if mask else slice(None) if not hdf5: # LOAD FILES if t: #TESTING FLAG from pycorr.gen_corrmat import fourD A_files = fourD + 7000 B_files = fourD + 7000 elif a and b: A_files = [os.path.join(a[0], fname) for fname in os.listdir(a[0])] #TODO change back, hack until rondo jobs are fixed B_files = [os.path.join(b[0], fname) for fname in os.listdir(b[0])] else: raise BaseException('need either test or specify inputs') A = [arr_slice(fname, ID)[...,mask].astype('float') for fname in A_files] B = [arr_slice(fname, ID)[...,mask].astype('float') for fname in B_files] # Thresholding #Hack to get threshold function, which is a class method TODO def move threshold import h5py Run = Run(h5py.File('dummy.h5')) # threshold tcs with low mean for dat in A+B: dat[Run.threshold(6000, dat)] = np.nan thresh_pass = [~np.isnan(dat.sum(axis=-1)) for dat in A+B] out['thresh_fail'] = Exp.cond_thresh(thresh_pass, mustpassprop=.7) else: E = Exp(hdf5) A = [run.load(use_subset=mask, standardized=True, threshold=True, _slice=ID) for run in E.iter_runs(a[0])] if b: #TODO fix, so hacky.. this script needs structure (want to let arg.b be optional B = [run.load(standardized=True, threshold=True, _slice=ID) for run in E.iter_runs(b[0])] else: B = [] E.get_cond(a[0]) out['thresh_fail'] = E.get_cond(a[0])['threshold'][...] # Combine group indices for correlation matrix (we will shuffle these) -------- indx_A = range(len(A)) indx_B = range(len(A), len(A + B)) print indx_A print indx_B # Cross-Correlation matrix (we will permute rows and columns) ----------------- out['isc_corrmat'] = crosscor(A+B, standardized=False) out['isc_A'] = intersubcorr(out['isc_corrmat'][..., indx_A, :][..., :, indx_A]) # Permutation Test ------------------------------------------------------------ if not isc_only: out_shape = (n_reps, ) + out['isc_corrmat'].shape[:-2] #n_reps x spatial_dims swap_dims = range(1,len(out_shape)) + [0] #list with first and last dims swapped out['null'] = perm(indx_A, indx_B, isc_corrmat_within_diff, C = out['isc_corrmat'], nreps=n_reps, out=np.zeros(out_shape)) out['null'] = out['null'].transpose(swap_dims) #put corrs on last dim out['r'] = isc_corrmat_within_diff(indx_A, indx_B, out['isc_corrmat'])[..., np.newaxis] #since 1 corr, add axis for broadcasting out['p'] = np.mean(np.abs(out['r']) <= np.abs(out['null']), axis=-1) # Output ---------------------------------------------------------------------- outtmp = os.path.join(outfile, "{fold}/{ID}.npy") for k, v in out.iteritems(): outfile = outtmp.format(fold=k, ID=x or ID) mkdir_p(os.path.dirname(outfile)) np.save(outfile, v)
from pycorr.pietools import mkdir_p from os.path import join import os.path basedir = os.path.dirname(os.path.abspath(__file__)) if len(sys.argv) > 1: example_dir = sys.argv[1] shutil.copy2(join(basedir, 'examples', example_dir, 'config.yaml'), 'config.yaml') shutil.copy2(join(basedir, 'tests/run_pipe.py'), 'run.py') os.symlink(join(basedir, 'pycorr/cli'), 'cli') if example_dir == 'full': print "setting up full example (using nifti files and ROIS)" import urllib, zipfile print "downloading data" datfile = 'pycorr_full_data.zip' datpath = os.path.join(basedir, 'examples', example_dir, datfile) # Get zipped data if needed, unzip in example folder if not os.path.isfile(datpath): urllib.urlretrieve('http://princeton.edu/~machow/pycorr_full_data.zip', datpath) zipfile.ZipFile(open(datpath, 'rb')).extractall() elif example_dir == 'test': import numpy as np from pycorr.gen_corrmat import fourD, fourD_sol mkdir_p('pipeline/subjects') for ii, sub in enumerate(fourD): np.save('pipeline/subjects/test_sub%s.npy'%ii, sub + 7000) np.save('pipeline/subjects/test_sol.npy', fourD_sol)
def permute_isc_within(a, b, x, outfile, mask='', isc_only=False, hdf5=None, thresh=6000, n_pass=.7, n_reps=1000, t=False): import numpy as np from pycorr.funcs_correlate import crosscor, intersubcorr from pycorr.statistics import perm, isc_corrmat_within_diff from pycorr.subject import Run, Exp from pycorr.pietools import mkdir_p, parse_section, arr_slice # TASK ID so script knows where to slice, converts SGE_TASK_ID to 0-indexed ID = parse_section(x) if x is not None else int( os.environ['SGE_TASK_ID']) - 1 # OUTPUTS out = {} # Load and Slice data --------------------------------------------------------- mask = np.load(mask) if mask else slice(None) if not hdf5: # LOAD FILES if t: #TESTING FLAG from pycorr.gen_corrmat import fourD A_files = fourD + 7000 B_files = fourD + 7000 elif a and b: A_files = [ os.path.join(a[0], fname) for fname in os.listdir(a[0]) ] #TODO change back, hack until rondo jobs are fixed B_files = [os.path.join(b[0], fname) for fname in os.listdir(b[0])] else: raise BaseException('need either test or specify inputs') A = [ arr_slice(fname, ID)[..., mask].astype('float') for fname in A_files ] B = [ arr_slice(fname, ID)[..., mask].astype('float') for fname in B_files ] # Thresholding #Hack to get threshold function, which is a class method TODO def move threshold import h5py Run = Run(h5py.File('dummy.h5')) # threshold tcs with low mean for dat in A + B: dat[Run.threshold(6000, dat)] = np.nan thresh_pass = [~np.isnan(dat.sum(axis=-1)) for dat in A + B] out['thresh_fail'] = Exp.cond_thresh(thresh_pass, mustpassprop=.7) else: E = Exp(hdf5) A = [ run.load(use_subset=mask, standardized=True, threshold=True, _slice=ID) for run in E.iter_runs(a[0]) ] if b: #TODO fix, so hacky.. this script needs structure (want to let arg.b be optional B = [ run.load(standardized=True, threshold=True, _slice=ID) for run in E.iter_runs(b[0]) ] else: B = [] E.get_cond(a[0]) out['thresh_fail'] = E.get_cond(a[0])['threshold'][...] # Combine group indices for correlation matrix (we will shuffle these) -------- indx_A = range(len(A)) indx_B = range(len(A), len(A + B)) print indx_A print indx_B # Cross-Correlation matrix (we will permute rows and columns) ----------------- out['isc_corrmat'] = crosscor(A + B, standardized=False) out['isc_A'] = intersubcorr(out['isc_corrmat'][..., indx_A, :][..., :, indx_A]) # Permutation Test ------------------------------------------------------------ if not isc_only: out_shape = ( n_reps, ) + out['isc_corrmat'].shape[:-2] #n_reps x spatial_dims swap_dims = range( 1, len(out_shape)) + [0] #list with first and last dims swapped out['null'] = perm(indx_A, indx_B, isc_corrmat_within_diff, C=out['isc_corrmat'], nreps=n_reps, out=np.zeros(out_shape)) out['null'] = out['null'].transpose(swap_dims) #put corrs on last dim out['r'] = isc_corrmat_within_diff(indx_A, indx_B, out['isc_corrmat'])[ ..., np.newaxis] #since 1 corr, add axis for broadcasting out['p'] = np.mean(np.abs(out['r']) <= np.abs(out['null']), axis=-1) # Output ---------------------------------------------------------------------- outtmp = os.path.join(outfile, "{fold}/{ID}.npy") for k, v in out.iteritems(): outfile = outtmp.format(fold=k, ID=x or ID) mkdir_p(os.path.dirname(outfile)) np.save(outfile, v)