def characterize_components_se(origTS_pc, data_mean, tes, t2s, S0, mmix, ICA_maps, voxelwiseQA,
Ncpus, ICA_maps_thr=0.0, discard_mask=None,writeOuts=False,outDir=None, outPrefix=None, mask=None, aff=None, head=None, Z_MAX=8, F_MAX=500, doFM=False, doMedian=False):
"""
This function computes kappa, rho and variance for each ICA component.
Parameters:
-----------
origTS_pc: original ME Timeseries in signal percent units for intra-cranial voxels (Nv,Ne,Nt)
data_mean: voxel-wise mean across time of the ME timeseries. (Nv,Ne)
tes: echo times (Ne,)
t2s: static T2* voxel-wise map (Nv,)
S0: static S0 voxel-wise map (Nv,)
mmix: ICA mixing matrix (Nc,Nt)
ICA_maps: ICA spatial maps (Nv,Nc)
voxelwiseQA: voxel-wise QA map obtained from attempting a static TE-dependence fit to the means. It
can be used as part of the wiegths used duing the averaging. (Nv,)
Ncpus: number of available CPUs for multi-processing.
ICA_maps_thr: threshold for selection of voxels entering the averaging for kappa and rho computation.
discard_mask: voxels to be discarded because the static field generated erroneous S0 or T2* values.
writeOuts: flag to instruct the function to save additional files.
outDir: path where the program should write NIFTI and other datasets.
outPrefix: prefix for datasets written to disk.
mask: binary map indicating intra-cranial voxels (Nv,)
aff: affine needed by nibabel to write NIFTI datasets.
head: header needed by nibabel to write NIFTI datasets.
Z_MAX: maximum allowed Z-score in ICA maps.
F_MAX: maximum allowed F-stat in R2 and S0 fits of the TE-dependence model.
Results:
--------
features: a 7xNc numpy array with features for each component. Column 0 is component ID,
column 1 is kappa, column 2 is rho, column 3 is explained variance, column 4 is
max Fstat in the R2 fit, column 5 is max Fstat in the S0 fit, column 6 is the
kappa/rho ratio. Components are sorted by variance (e.g., the way they came out of the ICA)
"""
Nv,Ne,Nt = origTS_pc.shape
Nc,_ = mmix.shape
# If no discard_mask is provided, create one in which no voxels will be discarded during the
# averaging.
if discard_mask is None:
discard_mask = np.zeros((Nv*Ne,), dtype=bool)
else:
discard_mask = np.tile(discard_mask,Ne)
# Get ICA-component masks based on threshold
ICA_maps_mask = np.zeros(ICA_maps.shape, dtype=bool) #(Nv*Ne,Nc)
ICA_maps_mask = np.logical_and((np.abs(ICA_maps)>ICA_maps_thr), discard_mask[:,np.newaxis] )
niiwrite_nv(np.reshape(ICA_maps_mask,(Nv,Ne,Nc),order='F'), mask,outDir+outPrefix+'.ICA.Zmaps.mask.nii',aff ,head)
# Compute overall variance in the ICA data
totalvar = (ICA_maps**2).sum() # Single Value
# Compute beta maps (fits to each echo)
# This is equivalent to running 3dDeconvolve with the mmix being the stim_files.
# I tried this and gives exactly the same result
# The 100 factor is there so that the b are kind of signal percent change
beta = 100*np.linalg.lstsq(mmix.T, (np.reshape(origTS_pc,(Nv*Ne,Nt))).T)[0].T
beta = np.reshape(beta,(Nv,Ne,Nc)) #(Nv,Ne,Nc)## <---------------------------------------- MAYBE PUT HERE AN ABS
#beta = np.reshape(ICA_maps,(Nv,Ne,Nc),order='F') #(Nv*Ne,Nc)
oc_ICA_maps= make_optcom(beta,t2s,tes)
oc_ICA_maps_mask = np.sum(np.reshape(ICA_maps_mask,(Nv,Ne,Nc),order='F'),axis=1)>(ceil(Ne/2.)) #(Nv,Nc)
niiwrite_nv(oc_ICA_maps_mask, mask,outDir+outPrefix+'.ICA.Zmaps.maskOC.nii',aff ,head) #(Nv,Nc)
# Initialize results holder
F_S0_maps = np.zeros((Nv,Nc))
F_R2_maps = np.zeros((Nv,Nc))
F_S0_masks = np.zeros((Nv,Nc), dtype=bool)
F_R2_masks = np.zeros((Nv,Nc), dtype=bool)
c_S0_maps = np.zeros((Nv,Nc))
c_R2_maps = np.zeros((Nv,Nc))
p_S0_maps = np.zeros((Nv,Nc))
p_R2_maps = np.zeros((Nv,Nc))
Kappa_maps = np.zeros((Nv,Nc))
Kappa_masks= np.zeros((Nv,Nc), dtype=bool)
Rho_maps = np.zeros((Nv,Nc))
Rho_masks = np.zeros((Nv,Nc), dtype=bool)
Weight_maps= np.zeros((Nv,Nc))
varexp = np.zeros(Nc)
kappas = np.zeros(Nc)
rhos = np.zeros(Nc)
if doFM:
FM_Slope_map = np.zeros((Nv,Nc))
FM_Inter_map = np.zeros((Nv,Nc))
FM_p_map = np.zeros((Nv,Nc))
FM_r_map = np.zeros((Nv,Nc))
FM_Slope_err_map = np.zeros((Nv,Nc))
FM_Slope_T_map = np.zeros((Nv,Nc))
FM_Slope_p_map = np.zeros((Nv,Nc))
# Compute metrics per component
X1 = np.ones((Ne,Nv)) #(Ne,Nv)
X2 = np.repeat((tes/tes.mean())[:,np.newaxis].T,Nv,axis=0).T #<-------------------- MAYBE NEEDS A MINUS SIGN (Ne,Nv)
print(" + Multi-process Characterize Components -> Ncpu = %d" % Ncpus)
pool = Pool(processes=Ncpus)
result = pool.map(_characterize_this_component_se, [{
'c': c, 'F_MAX':F_MAX, 'Z_MAX':Z_MAX,
'X1': X1, 'X2':X2,
'aff': aff, 'head':head, 'outDir':outDir, 'outPrefix':outPrefix,
'mask': mask,
'B': np.atleast_3d(beta)[:,:,c].transpose(),
'weight_map': (oc_ICA_maps[:,c]**2.)*voxelwiseQA,
'c_mask': oc_ICA_maps_mask[:,c],
'writeOuts': writeOuts,
'doFM': doFM, 'doMedian':doMedian
} for c in np.arange(Nc)])
feat_names = ['cID','Kappa','Rho','Var','maxFR2','maxFS0','K/R', 'maxZICA','NvZmask','NvFR2mask','NvFS0mask','NvKapMask','NvRhoMask','Dan']
feat_vals = np.zeros((Nc,len(feat_names)))
for c in range(Nc):
Weight_maps[:,c] = (oc_ICA_maps[:,c]**2.)*voxelwiseQA
Kappa_maps[:,c] = result[c]['Kappa_map']
Rho_maps[:,c] = result[c]['Rho_map']
F_S0_maps[:,c] = result[c]['FS0']
F_R2_maps[:,c] = result[c]['FR2']
c_S0_maps[:,c] = result[c]['cS0']
c_R2_maps[:,c] = result[c]['cR2']
p_S0_maps[:,c] = result[c]['pS0']
p_R2_maps[:,c] = result[c]['pR2']
F_S0_masks[:,c] = result[c]['F_S0_mask']
F_R2_masks[:,c] = result[c]['F_R2_mask']
Kappa_masks[:,c] = result[c]['Kappa_mask']
Rho_masks[:,c] = result[c]['Rho_mask']
if doFM:
FM_Slope_map[:,c] = result[c]['FM_Slope']
FM_Inter_map[:,c] = result[c]['FM_Inter']
FM_p_map[:,c] = result[c]['FM_p']
FM_r_map[:,c] = result[c]['FM_r']
FM_Slope_err_map[:,c] = result[c]['FM_Slope_err']
FM_Slope_T_map[:,c] = result[c]['FM_Slope_T']
FM_Slope_p_map[:,c] = result[c]['FM_Slope_p']
feat_vals[c,0] = c
feat_vals[c,1] = result[c]['Kappa']
feat_vals[c,2] = result[c]['Rho']
feat_vals[c,3] = 100*((ICA_maps[:,c]**2).sum()/totalvar)
FR2_mask_arr = ma.masked_array(F_R2_maps[:,c], mask=np.logical_not(Kappa_masks[:,c])) #abs(Kappa_masks[:,c]-1))
feat_vals[c,4] = FR2_mask_arr.max()
FS0_mask_arr = ma.masked_array(F_S0_maps[:,c], mask=np.logical_not(Rho_masks[:,c])) #abs(Rho_masks[:,c]-1))
feat_vals[c,5] = FS0_mask_arr.max()
feat_vals[c,6] = feat_vals[c,1] / feat_vals[c,2]
ZICA_mask_arr = ma.masked_array(oc_ICA_maps[:,c], mask=np.logical_not(oc_ICA_maps_mask[:,c])) #abs(ICA_maps_mask[:,c]-1))
feat_vals[c,7] = ZICA_mask_arr.max()
feat_vals[c,8] = ICA_maps_mask[:,c].sum()
feat_vals[c,9] = F_R2_masks[:,c].sum()
feat_vals[c,10] = F_S0_masks[:,c].sum()
feat_vals[c,11] = Kappa_masks[:,c].sum()
feat_vals[c,12] = Rho_masks[:,c].sum()
# DAN METRIC
if doFM:
aux_mask = np.logical_and((FM_p_map[:,c]<0.05),(FM_Slope_p_map[:,c]<0.05))
aux_numerator = Weight_maps[aux_mask,c].sum()
aux_denominat = Weight_maps[:,c].sum()
aux_metric = aux_numerator/aux_denominat
print("[%d] -> aux_mask%s | aux_numerator%s | DF=%f" % (c,str(aux_mask.shape),str(Weight_maps[aux_mask,c].shape),aux_metric))
df_feats = pd.DataFrame(data=feat_vals,columns=feat_names)
df_feats.to_csv(outDir+outPrefix+'.DF.csv')
df_feats['cID'] = df_feats['cID'].astype(int)
niiwrite_nv(beta , mask,outDir+outPrefix+'.chComp.Beta.nii',aff ,head)
niiwrite_nv(F_S0_maps , mask,outDir+outPrefix+'.chComp.FS0.nii',aff ,head)
niiwrite_nv(F_R2_maps , mask,outDir+outPrefix+'.chComp.FR2.nii',aff ,head)
niiwrite_nv(F_S0_masks, mask,outDir+outPrefix+'.chComp.FS0.mask.nii',aff ,head)
niiwrite_nv(F_R2_masks, mask,outDir+outPrefix+'.chComp.FR2.mask.nii',aff ,head)
niiwrite_nv(c_S0_maps , mask,outDir+outPrefix+'.chComp.cS0.nii',aff ,head)
niiwrite_nv(c_R2_maps , mask,outDir+outPrefix+'.chComp.cR2.nii',aff ,head)
niiwrite_nv(p_S0_maps , mask,outDir+outPrefix+'.chComp.pS0.nii',aff ,head)
niiwrite_nv(p_R2_maps , mask,outDir+outPrefix+'.chComp.pR2.nii',aff ,head)
niiwrite_nv(Kappa_maps, mask,outDir+outPrefix+'.chComp.Kappa.nii',aff ,head)
niiwrite_nv(Kappa_masks, mask,outDir+outPrefix+'.chComp.Kappa_mask.nii',aff ,head)
niiwrite_nv(Rho_masks, mask,outDir+outPrefix+'.chComp.Rho_mask.nii',aff ,head)
niiwrite_nv(Rho_maps, mask,outDir+outPrefix+'.chComp.Rho.nii',aff ,head)
niiwrite_nv(Weight_maps, mask,outDir+outPrefix+'.chComp.weightMaps.nii',aff ,head)
if doFM:
niiwrite_nv(FM_Slope_map , mask,outDir+outPrefix+'.chComp.FM.Slope.nii',aff ,head)
niiwrite_nv(FM_Inter_map , mask,outDir+outPrefix+'.chComp.FM.Inter.nii',aff ,head)
niiwrite_nv(FM_p_map , mask,outDir+outPrefix+'.chComp.FM.p.nii',aff ,head)
niiwrite_nv(FM_r_map , mask,outDir+outPrefix+'.chComp.FM.r.nii',aff ,head)
niiwrite_nv(FM_Slope_err_map , mask,outDir+outPrefix+'.chComp.FM.Slope.err.nii',aff ,head)
niiwrite_nv(FM_Slope_T_map , mask,outDir+outPrefix+'.chComp.FM.Slope.T.nii',aff ,head)
niiwrite_nv(FM_Slope_p_map , mask,outDir+outPrefix+'.chComp.FM.Slope.p.nii',aff ,head)
return df_feats
# ---------------------
origMask_needed = True
origMask_path = os.path.join(outputDir, options.prefix + '.mask.orig.nii')
if options.reuse:
if os.path.exists(origMask_path):
print("++ INFO [Main]: Re-using existing original mask [%s]" %
(origMask_path))
mask, _, _ = meu.niiLoad(origMask_path)
mask = (mask > 0)
origMask_needed = False
if origMask_needed:
if options.mask_file == None:
print("++ INFO [Main]: Generating initial mask from data.")
mask = meu.mask4MEdata(mepi_data)
meu.niiwrite_nv(
mask[mask], mask,
options.out_dir + options.prefix + '.mask.orig.nii', mepi_aff,
mepi_head)
else:
print("++ INFO [Main]: Using user-provided mask.")
if not os.path.exists(options.mask_file):
print("++ Error: Provided mask [%s] does not exist." %
options.mask_file)
sys.exit()
mask, _, _ = meu.niiLoad(options.mask_file)
mask = (mask > 0)
Nv = np.sum(mask)
print(" + Number of Voxels in mask [Nv=%i]" % Nv)
# Reshape the input data
# ----------------------
SME = mepi_data[mask, :, :].astype(float) #(Nv,Ne,Nt)
def _characterize_this_component_se(item):
B = item['B'] #(Ne,Nv)
X1 = item['X1'] #(Ne,Nv)
X2 = item['X2'] #(Ne,Nv)
aff = item['aff']
head = item['head']
mask = item['mask']
c = item['c']
outDir = item['outDir']
outPrefix = item['outPrefix']
F_MAX = item['F_MAX']
Z_MAX = item['Z_MAX']
weight_map = item['weight_map']
c_mask = item['c_mask']
writeOuts = item['writeOuts']
doFM = item['doFM']
doMedian = item['doMedian']
Ne, Nv = B.shape
Kappa_mask = c_mask.copy()
Rho_mask = c_mask.copy()
# Write the fits for the differente echoes into file (very useful for debugging purposes)
if writeOuts:
niiwrite_nv(B.T,mask,outDir+outPrefix+'.chComp.EXTRA.Beta'+str(c).zfill(3)+'.nii',aff,head)
# S0 Model
coeffs_S0 = (B*X1).sum(axis=0)/(X1**2).sum(axis=0) #(Nv,)
estima_S0 = X1*np.tile(coeffs_S0,(Ne,1))
SSR_S0 = (estima_S0**2).sum(axis=0)
SSE_S0 = ((B-estima_S0)**2).sum(axis=0)
dofSSR_S0 = 1
dofSSE_S0 = Ne -1
F_S0 = (SSR_S0/dofSSR_S0) / (SSE_S0/dofSSE_S0) #(Nv,)
p_S0 = 1-f.cdf(F_S0,1,Ne-1)
F_S0_mask = (p_S0 < 0.05)
F_S0[F_S0>F_MAX] = F_MAX
F_S0_AllValues = F_S0.copy()
if writeOuts:
niiwrite_nv((X1*np.tile(coeffs_S0,(Ne,1))).T,mask,outDir+outPrefix+'.chComp.EXTRA.S0Fit'+str(c).zfill(3)+'.nii',aff,head)
# R2 Model
# If beta_i = alpha_i*TE/mean(TE) + error --> To solve this RTO model (Regression Through the Origin), it is possible to obtain
# the alpha_i that provides the best fit (in a least-squares fashion) by simply using the equation below
# Please look at: Eishenhouer JG "Regression throught the origin" Technical Statistics (25):3, 2003
coeffs_R2 = (B*X2).sum(axis=0)/(X2**2).sum(axis=0) #(Nv,)
estima_R2 = X2*np.tile(coeffs_R2,(Ne,1))
SSR_R2 = (estima_R2**2).sum(axis=0)
SSE_R2 = ((B-estima_R2)**2).sum(axis=0)
dofSSR_R2 = 1
dofSSE_R2 = Ne -1
F_R2 = (SSR_R2/dofSSR_R2) / (SSE_R2/dofSSE_R2) #(Nv,)
p_R2 = 1-f.cdf(F_R2,1,Ne-1)
F_R2_mask = (p_R2 < 0.05)
F_R2[F_R2>F_MAX] = F_MAX
F_R2_AllValues = F_R2.copy()
if writeOuts:
niiwrite_nv((X2*np.tile(coeffs_R2,(Ne,1))).T,mask,outDir+outPrefix+'.chComp.EXTRA.R2Fit'+str(c).zfill(3)+'.nii',aff,head)
# Mask for spatial averaging
Kappa_mask = np.logical_and(c_mask, np.logical_or(F_R2_mask, F_S0_mask)) #(Nv,)
Rho_mask = Kappa_mask.copy() #(Nv,)
# Kappa Computation
Kappa_map = F_R2 * weight_map
Kappa_map = Kappa_map/(weight_map[Kappa_mask].mean())
Kappa_map = Kappa_map * Kappa_mask # weigths from the voxels entering the computation
if doMedian:
Kappa = np.median(Kappa_map[Kappa_mask])
else:
Kappa = np.mean(Kappa_map[Kappa_mask])
# Rho Computation
Rho_map = F_S0 * weight_map
Rho_map = Rho_map/(weight_map[Rho_mask].mean())
Rho_map = Rho_map * Rho_mask # weigths from the voxels entering the computation
if doMedian:
Rho = np.median(Rho_map[Rho_mask])
else:
Rho = np.mean(Rho_map[Rho_mask])
# EXTRA CODE
if doFM==True:
FullModel_Slope = np.zeros((Nv,))
FullModel_Inter = np.zeros((Nv,))
FullModel_p = np.zeros((Nv,))
FullModel_r = np.zeros((Nv,))
FullModel_Slope_err = np.zeros((Nv,))
FullModel_Slope_T = np.zeros((Nv,))
FullModel_Slope_p = np.zeros((Nv,))
for v in range(Nv):
FullModel_Slope[v], FullModel_Inter[v], FullModel_r[v], FullModel_p[v], FullModel_Slope_err[v] = linregress(X2[:,v],B[:,v])
FullModel_Slope_T[v] = FullModel_Slope[v]/FullModel_Slope_err[v]
FullModel_Slope_p[v] = 2*(1-t.cdf(np.abs(FullModel_Slope_T[v]),Ne-2))
return {'Kappa':Kappa, 'Rho':Rho, 'Kappa_map':Kappa_map, 'Rho_map':Rho_map, 'FS0':F_S0_AllValues, 'FR2':F_R2_AllValues, 'cS0':coeffs_S0, 'cR2':coeffs_R2,
'Kappa_mask':Kappa_mask,'Rho_mask':Rho_mask, 'F_R2_mask':F_R2_mask, 'F_S0_mask':F_S0_mask, 'pR2':p_R2, 'pS0':p_S0,
'FM_Slope':FullModel_Slope, 'FM_Inter':FullModel_Inter, 'FM_p':FullModel_p, 'FM_r':FullModel_r,
'FM_Slope_err':FullModel_Slope_err, 'FM_Slope_T':FullModel_Slope_T, 'FM_Slope_p':FullModel_Slope_p}
else:
return {'Kappa':Kappa, 'Rho':Rho, 'Kappa_map':Kappa_map, 'Rho_map':Rho_map, 'FS0':F_S0_AllValues, 'FR2':F_R2_AllValues, 'cS0':coeffs_S0, 'cR2':coeffs_R2,
'Kappa_mask':Kappa_mask,'Rho_mask':Rho_mask, 'F_R2_mask':F_R2_mask, 'F_S0_mask':F_S0_mask, 'pR2':p_R2, 'pS0':p_S0}