def crosscor_full(A, B=None, nan_thresh=None):
"""From data (dims sub x seg x vox) calculate sub against others pattern cor
Parameters:
A: sub x seg x vox matrix
B: (optional) seg x vox matrix
Returns:
seg x seg correlation matrix
"""
# standardize all along last dim, so don't need to in correlation
A = standardize(A)
all_cors = []
# Within group correlations
if B is None:
others = sum_tc(A)
for sub in A:
# check for nan
to_remove = np.any(np.isnan(sub), axis=0)
if np.any(to_remove):
tmp_sub = sub[..., to_remove]
tmp_others = others[..., to_remove]
else:
tmp_sub, tmp_others = sub, others
# cross correlate (1 x seg x seg)
if nan_thresh is not None and to_remove.mean() > nan_thresh:
cormat = np.empty(sub.shape[0:1] * 2) * np.nan
else:
cormat = crosscor(tmp_sub,
standardize(tmp_others - tmp_sub),
standardized=True)
all_cors.append(cormat)
return np.array(all_cors)
# Between group correlations
else:
B = standardize(B)
for sub in A:
cormat = crosscor(sub, B, standardized=True)
all_cors.append(cormat)
return np.array(all_cors)
def crosscor_full(A, B=None, nan_thresh=None):
"""From data (dims sub x seg x vox) calculate sub against others pattern cor
Parameters:
A: sub x seg x vox matrix
B: (optional) seg x vox matrix
Returns:
seg x seg correlation matrix
"""
# standardize all along last dim, so don't need to in correlation
A = standardize(A)
all_cors = []
# Within group correlations
if B is None:
others = sum_tc(A)
for sub in A:
# check for nan
to_remove = np.any(np.isnan(sub), axis=0)
if np.any(to_remove):
tmp_sub = sub[...,to_remove]
tmp_others = others[..., to_remove]
else:
tmp_sub, tmp_others = sub, others
# cross correlate (1 x seg x seg)
if nan_thresh is not None and to_remove.mean() > nan_thresh:
cormat = np.empty(sub.shape[0:1]*2) * np.nan
else:
cormat = crosscor(tmp_sub, standardize(tmp_others - tmp_sub), standardized=True)
all_cors.append(cormat)
return np.array(all_cors)
# Between group correlations
else:
B = standardize(B)
for sub in A:
cormat = crosscor(sub, B, standardized=True)
all_cors.append(cormat)
return np.array(all_cors)
def permute_isc_within(a, b, x, outfile, mask='', isc_only=False, hdf5=None, thresh=6000, n_pass=.7, n_reps=1000, t=False):
import numpy as np
from pycorr.funcs_correlate import crosscor, intersubcorr
from pycorr.statistics import perm, isc_corrmat_within_diff
from pycorr.subject import Run, Exp
from pycorr.pietools import mkdir_p, parse_section, arr_slice
# TASK ID so script knows where to slice, converts SGE_TASK_ID to 0-indexed
ID = parse_section(x) if x is not None else int(os.environ['SGE_TASK_ID']) - 1
# OUTPUTS
out = {}
# Load and Slice data ---------------------------------------------------------
mask = np.load(mask) if mask else slice(None)
if not hdf5:
# LOAD FILES
if t: #TESTING FLAG
from pycorr.gen_corrmat import fourD
A_files = fourD + 7000
B_files = fourD + 7000
elif a and b:
A_files = [os.path.join(a[0], fname) for fname in os.listdir(a[0])] #TODO change back, hack until rondo jobs are fixed
B_files = [os.path.join(b[0], fname) for fname in os.listdir(b[0])]
else: raise BaseException('need either test or specify inputs')
A = [arr_slice(fname, ID)[...,mask].astype('float') for fname in A_files]
B = [arr_slice(fname, ID)[...,mask].astype('float') for fname in B_files]
# Thresholding
#Hack to get threshold function, which is a class method TODO def move threshold
import h5py
Run = Run(h5py.File('dummy.h5'))
# threshold tcs with low mean
for dat in A+B: dat[Run.threshold(6000, dat)] = np.nan
thresh_pass = [~np.isnan(dat.sum(axis=-1)) for dat in A+B]
out['thresh_fail'] = Exp.cond_thresh(thresh_pass, mustpassprop=.7)
else:
E = Exp(hdf5)
A = [run.load(use_subset=mask, standardized=True, threshold=True, _slice=ID) for run in E.iter_runs(a[0])]
if b: #TODO fix, so hacky.. this script needs structure (want to let arg.b be optional
B = [run.load(standardized=True, threshold=True, _slice=ID) for run in E.iter_runs(b[0])]
else: B = []
E.get_cond(a[0])
out['thresh_fail'] = E.get_cond(a[0])['threshold'][...]
# Combine group indices for correlation matrix (we will shuffle these) --------
indx_A = range(len(A))
indx_B = range(len(A), len(A + B))
print indx_A
print indx_B
# Cross-Correlation matrix (we will permute rows and columns) -----------------
out['isc_corrmat'] = crosscor(A+B, standardized=False)
out['isc_A'] = intersubcorr(out['isc_corrmat'][..., indx_A, :][..., :, indx_A])
# Permutation Test ------------------------------------------------------------
if not isc_only:
out_shape = (n_reps, ) + out['isc_corrmat'].shape[:-2] #n_reps x spatial_dims
swap_dims = range(1,len(out_shape)) + [0] #list with first and last dims swapped
out['null'] = perm(indx_A, indx_B, isc_corrmat_within_diff, C = out['isc_corrmat'],
nreps=n_reps, out=np.zeros(out_shape))
out['null'] = out['null'].transpose(swap_dims) #put corrs on last dim
out['r'] = isc_corrmat_within_diff(indx_A, indx_B, out['isc_corrmat'])[..., np.newaxis] #since 1 corr, add axis for broadcasting
out['p'] = np.mean(np.abs(out['r']) <= np.abs(out['null']), axis=-1)
# Output ----------------------------------------------------------------------
outtmp = os.path.join(outfile, "{fold}/{ID}.npy")
for k, v in out.iteritems():
outfile = outtmp.format(fold=k, ID=x or ID)
mkdir_p(os.path.dirname(outfile))
np.save(outfile, v)
def permute_isc_within(a,
b,
x,
outfile,
mask='',
isc_only=False,
hdf5=None,
thresh=6000,
n_pass=.7,
n_reps=1000,
t=False):
import numpy as np
from pycorr.funcs_correlate import crosscor, intersubcorr
from pycorr.statistics import perm, isc_corrmat_within_diff
from pycorr.subject import Run, Exp
from pycorr.pietools import mkdir_p, parse_section, arr_slice
# TASK ID so script knows where to slice, converts SGE_TASK_ID to 0-indexed
ID = parse_section(x) if x is not None else int(
os.environ['SGE_TASK_ID']) - 1
# OUTPUTS
out = {}
# Load and Slice data ---------------------------------------------------------
mask = np.load(mask) if mask else slice(None)
if not hdf5:
# LOAD FILES
if t: #TESTING FLAG
from pycorr.gen_corrmat import fourD
A_files = fourD + 7000
B_files = fourD + 7000
elif a and b:
A_files = [
os.path.join(a[0], fname) for fname in os.listdir(a[0])
] #TODO change back, hack until rondo jobs are fixed
B_files = [os.path.join(b[0], fname) for fname in os.listdir(b[0])]
else:
raise BaseException('need either test or specify inputs')
A = [
arr_slice(fname, ID)[..., mask].astype('float')
for fname in A_files
]
B = [
arr_slice(fname, ID)[..., mask].astype('float')
for fname in B_files
]
# Thresholding
#Hack to get threshold function, which is a class method TODO def move threshold
import h5py
Run = Run(h5py.File('dummy.h5'))
# threshold tcs with low mean
for dat in A + B:
dat[Run.threshold(6000, dat)] = np.nan
thresh_pass = [~np.isnan(dat.sum(axis=-1)) for dat in A + B]
out['thresh_fail'] = Exp.cond_thresh(thresh_pass, mustpassprop=.7)
else:
E = Exp(hdf5)
A = [
run.load(use_subset=mask,
standardized=True,
threshold=True,
_slice=ID) for run in E.iter_runs(a[0])
]
if b: #TODO fix, so hacky.. this script needs structure (want to let arg.b be optional
B = [
run.load(standardized=True, threshold=True, _slice=ID)
for run in E.iter_runs(b[0])
]
else:
B = []
E.get_cond(a[0])
out['thresh_fail'] = E.get_cond(a[0])['threshold'][...]
# Combine group indices for correlation matrix (we will shuffle these) --------
indx_A = range(len(A))
indx_B = range(len(A), len(A + B))
print indx_A
print indx_B
# Cross-Correlation matrix (we will permute rows and columns) -----------------
out['isc_corrmat'] = crosscor(A + B, standardized=False)
out['isc_A'] = intersubcorr(out['isc_corrmat'][..., indx_A, :][..., :,
indx_A])
# Permutation Test ------------------------------------------------------------
if not isc_only:
out_shape = (
n_reps, ) + out['isc_corrmat'].shape[:-2] #n_reps x spatial_dims
swap_dims = range(
1, len(out_shape)) + [0] #list with first and last dims swapped
out['null'] = perm(indx_A,
indx_B,
isc_corrmat_within_diff,
C=out['isc_corrmat'],
nreps=n_reps,
out=np.zeros(out_shape))
out['null'] = out['null'].transpose(swap_dims) #put corrs on last dim
out['r'] = isc_corrmat_within_diff(indx_A, indx_B, out['isc_corrmat'])[
..., np.newaxis] #since 1 corr, add axis for broadcasting
out['p'] = np.mean(np.abs(out['r']) <= np.abs(out['null']), axis=-1)
# Output ----------------------------------------------------------------------
outtmp = os.path.join(outfile, "{fold}/{ID}.npy")
for k, v in out.iteritems():
outfile = outtmp.format(fold=k, ID=x or ID)
mkdir_p(os.path.dirname(outfile))
np.save(outfile, v)
corrs = np.arange(-.4, 1.1, .1)
nsubs = 3
npoints = 10
ttl_pnts = nsubs * len(corrs) * 3 * npoints
out = []
sol = []
for lam in corrs:
C, data = corr_eig(lam, nsubs, npoints)
#print np.corrcoef(data.T)
sol.append(C)
out.append(data)
data
from pycorr.funcs_correlate import crosscor
out = np.array(out).transpose([2, 0, 1])
out_sol = crosscor(out, standardized=False)
np.allclose(out_sol, np.array(sol))
#for ii, sub in enumerate(out): print ii
fourD = out[[(0, 1, 2) * 2 * 2]].reshape([3, 2, 2, 15, 10])
fourD_sol = np.concatenate([sol] * 4).reshape([2, 2, 15, 3, 3])
# subject dim x (indx 1 is below thresh) x (indx 1 has two nan subs) x (time)
out_match = fourD + 7000
out_nan = fourD.copy() #nan for two subjects
out_nan[0:2, ...] = np.nan
out_thresh = fourD.copy() #below thresh
#np.allclose(crosscor(out_match), out_sol)
#np.allclose(crosscor(fourD, standardized=False), out_sol)
from nose import with_setup
import numpy as np
from numpy.testing import assert_almost_equal
from pycorr.funcs_correlate import standardize, corsubs, crosscor, intersubcorr
np.random.seed(10)
dims = (2,2, 10)
nsubs = 3
subs = [np.random.random(dims) for ii in range(nsubs)]
for M in subs: M[0,0] = range(dims[-1]) #0,0 is 1:N
for M in subs: standardize(M, inplace=True)
subs[0][1,1] = np.NAN #1,1 sub 0 has a NaN timecourse
C_all = crosscor(subs, standardized=True)
C_all[1,1,0] = np.NAN
isc1 = intersubcorr(C_all)
M_ttl = np.nansum(subs, axis=0)
isc2 = np.array([corsubs(M, M_ttl-M) for M in subs]).transpose([1,2,0])
isc3_list = []
for M in subs:
r_all = corsubs(M, M_ttl)
s_all = np.std(M_ttl, axis=-1, ddof=1)
s_i = np.std(M, axis=-1, ddof=1)
M_cors = (r_all*s_all - s_i) / \
np.sqrt(s_i**2 + s_all**2 - 2*s_i*s_all*r_all) #wherry formula
isc3_list.append(M_cors)
isc3 = np.array(isc3_list).transpose([1,2,0])
corrs = np.arange(-.4,1.1,.1)
nsubs = 3
npoints = 10
ttl_pnts = nsubs * len(corrs)* 3 * npoints
out = []
sol = []
for lam in corrs:
C, data = corr_eig(lam, nsubs, npoints)
#print np.corrcoef(data.T)
sol.append(C)
out.append(data)
data
from pycorr.funcs_correlate import crosscor
out = np.array(out).transpose([2,0,1])
out_sol = crosscor(out, standardized=False)
np.allclose(out_sol, np.array(sol))
#for ii, sub in enumerate(out): print ii
fourD = out[[(0,1,2)*2*2]].reshape([3,2,2,15,10])
fourD_sol = np.concatenate([sol]*4).reshape([2,2,15,3,3])
# subject dim x (indx 1 is below thresh) x (indx 1 has two nan subs) x (time)
out_match = fourD + 7000
out_nan = fourD.copy() #nan for two subjects
out_nan[0:2,...] = np.nan
out_thresh = fourD.copy() #below thresh
#np.allclose(crosscor(out_match), out_sol)
#np.allclose(crosscor(fourD, standardized=False), out_sol)
import numpy as np
from numpy.testing import assert_almost_equal
from pycorr.funcs_correlate import standardize, corsubs, crosscor, intersubcorr
np.random.seed(10)
dims = (2, 2, 10)
nsubs = 3
subs = [np.random.random(dims) for ii in range(nsubs)]
for M in subs:
M[0, 0] = range(dims[-1]) #0,0 is 1:N
for M in subs:
standardize(M, inplace=True)
subs[0][1, 1] = np.NAN #1,1 sub 0 has a NaN timecourse
C_all = crosscor(subs, standardized=True)
C_all[1, 1, 0] = np.NAN
isc1 = intersubcorr(C_all)
M_ttl = np.nansum(subs, axis=0)
isc2 = np.array([corsubs(M, M_ttl - M) for M in subs]).transpose([1, 2, 0])
isc3_list = []
for M in subs:
r_all = corsubs(M, M_ttl)
s_all = np.std(M_ttl, axis=-1, ddof=1)
s_i = np.std(M, axis=-1, ddof=1)
M_cors = (r_all*s_all - s_i) / \
np.sqrt(s_i**2 + s_all**2 - 2*s_i*s_all*r_all) #wherry formula
isc3_list.append(M_cors)
isc3 = np.array(isc3_list).transpose([1, 2, 0])
