def map_beta(censor, coeff_df, power2011):
# create a left censor
censor_left = censor.copy()
censor_left['beta'] = coeff_df
# make symmetric censor df
censor_right = censor_left[['connID', 'beta']].copy()
censor_right['connID'] = censor_right['connID'].apply(reverse_connID)
censor_LR = censor_left.append(censor_right, ignore_index=True)
censor_LR = censor_LR[['connID', 'beta']]
# merge to main roi df
roi_df = get_ROI_df(power2011)
roi_df = roi_df.merge(censor_LR, how="left", on=['connID'])
roi_df['beta'] = roi_df['beta'].replace(np.nan, 0.0)
# reformat beta matrix
adj_vector = roi_df['beta'].values
adj_beta = pd.DataFrame(vector2matrix(adj_vector), dtype='float')
# grab center coordinates for atlas labels
coor_vector = np.array([(0, 0, 0) for i in range(264)])
power = datasets.fetch_coords_power_2011()
power_coords = np.vstack(
(power.rois['x'], power.rois['y'], power.rois['z'])).T
return adj_beta, power_coords
def run_analyses(
files,
qc,
out_dir=".",
confounds=None,
n_iters=10000,
n_jobs=1,
qc_thresh=0.2,
window=1000,
):
"""Run scrubbing, high-low motion, and QCRSFC analyses.
Parameters
----------
files : (N,) list of nifti files
List of 4D (X x Y x Z x T) images in MNI space.
qc : (N,) list of array_like
List of 1D (T) numpy arrays with QC metric values per img (e.g., FD or respiration).
out_dir : str, optional
Output directory. Default is current directory.
confounds : None or (N,) list of array-like, optional
List of 2D (T) numpy arrays with confounds per img.
Default is None (no confounds are removed).
n_iters : int, optional
Number of iterations to run to generate null distributions. Default is 10000.
n_jobs : int, optional
The number of CPUs to use to do the computation. -1 means 'all CPUs'. Default is 1.
qc_thresh : float, optional
Threshold for QC metric used in scrubbing analysis. Default is 0.2 (for FD).
window : int, optional
Number of units (pairs of ROIs) to include when averaging to generate smoothing curve.
Default is 1000.
Notes
-----
This function writes out several files to out_dir:
- ``analysis_values.tsv.gz``: Raw analysis values for analyses.
Has four columns: distance, qcrsfc, scrubbing, and highlow.
- ``smoothing_curves.tsv.gz``: Smoothing curve information for analyses.
Has four columns: distance, qcrsfc, scrubbing, and highlow.
- ``null_smoothing_curves.npz``:
Null smoothing curves from each analysis.
Contains three 2D arrays, where number of columns is same
size and order as distance column in ``smoothing_curves.tsv.gz``
and number of rows is number of iterations for permutation analysis.
The three arrays' keys are 'qcrsfc', 'highlow', and 'scrubbing'.
- ``[analysis]_analysis.png``: Figure for each analysis.
"""
makedirs(out_dir, exist_ok=True)
# create LGR with 'spam_application'
LGR.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler(op.join(out_dir, "log.tsv"))
fh.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter(
"%(asctime)s\t%(name)-12s\t%(levelname)-8s\t%(message)s")
fh.setFormatter(formatter)
# add the handlers to the LGR
LGR.addHandler(fh)
LGR.info("Preallocating matrices")
n_subjects = len(files)
# Load atlas and associated masker
atlas = datasets.fetch_coords_power_2011()
coords = np.vstack((atlas.rois["x"], atlas.rois["y"], atlas.rois["z"])).T
n_rois = coords.shape[0]
triu_idx = np.triu_indices(n_rois, k=1)
distances = squareform(pdist(coords))
distances = distances[triu_idx]
# Sorting index for distances
edge_sorting_idx = distances.argsort()
distances = distances[edge_sorting_idx]
LGR.info("Creating masker")
spheres_masker = input_data.NiftiSpheresMasker(
seeds=coords,
radius=5.0,
t_r=None,
smoothing_fwhm=4.0,
detrend=False,
standardize=False,
low_pass=None,
high_pass=None,
)
# prep for qcrsfc and high-low motion analyses
mean_qc = np.array([np.mean(subj_qc) for subj_qc in qc])
z_corr_mats = np.zeros((n_subjects, distances.size))
# Get correlation matrices
ts_all = []
LGR.info("Building correlation matrices")
if confounds:
LGR.info("Regressing confounds out of data.")
for i_subj in range(n_subjects):
if confounds:
raw_ts = spheres_masker.fit_transform(
files[i_subj], confounds=confounds[i_subj]).T
else:
raw_ts = spheres_masker.fit_transform(files[i_subj]).T
assert raw_ts.shape[0] == n_rois
if np.any(np.isnan(raw_ts)):
raise ValueError(f"Time series of {files[i_subj]} contains NaNs")
roi_variances = np.var(raw_ts, axis=1)
if any(roi_variances == 0):
bad_rois = np.where(roi_variances == 0)[0]
raise ValueError(
f"ROI(s) {bad_rois} for {files[i_subj]} have variance of 0.")
ts_all.append(raw_ts)
raw_corrs = np.corrcoef(raw_ts)
raw_corrs = raw_corrs[triu_idx]
raw_corrs = raw_corrs[
edge_sorting_idx] # Sort from close to far ROI pairs
z_corr_mats[i_subj, :] = np.arctanh(raw_corrs)
del (raw_corrs, raw_ts, spheres_masker, atlas, coords)
analysis_values = pd.DataFrame(columns=["qcrsfc", "highlow", "scrubbing"],
index=distances)
analysis_values.index.name = "distance"
# QC:RSFC r analysis
LGR.info("Performing QC:RSFC analysis")
qcrsfc_values = analysis.qcrsfc_analysis(mean_qc, z_corr_mats)
analysis_values["qcrsfc"] = qcrsfc_values
qcrsfc_smoothing_curve = utils.moving_average(qcrsfc_values, window)
qcrsfc_smoothing_curve, smoothing_curve_distances = utils.average_across_distances(
qcrsfc_smoothing_curve,
distances,
)
# Quick interlude to create the smoothing_curves DataFrame
smoothing_curves = pd.DataFrame(
columns=["qcrsfc", "highlow", "scrubbing"],
index=smoothing_curve_distances,
)
smoothing_curves.index.name = "distance"
smoothing_curves.loc[smoothing_curve_distances,
"qcrsfc"] = qcrsfc_smoothing_curve
del qcrsfc_values, qcrsfc_smoothing_curve
# High-low motion analysis
LGR.info("Performing high-low motion analysis")
highlow_values = analysis.highlow_analysis(mean_qc, z_corr_mats)
analysis_values["highlow"] = highlow_values
hl_smoothing_curve = utils.moving_average(highlow_values, window)
hl_smoothing_curve, smoothing_curve_distances = utils.average_across_distances(
hl_smoothing_curve,
distances,
)
smoothing_curves.loc[smoothing_curve_distances,
"highlow"] = hl_smoothing_curve
del highlow_values, hl_smoothing_curve
# Scrubbing analysis
LGR.info("Performing scrubbing analysis")
scrub_values = analysis.scrubbing_analysis(qc,
ts_all,
edge_sorting_idx,
qc_thresh,
perm=False)
analysis_values["scrubbing"] = scrub_values
scrub_smoothing_curve = utils.moving_average(scrub_values, window)
scrub_smoothing_curve, smoothing_curve_distances = utils.average_across_distances(
scrub_smoothing_curve,
distances,
)
smoothing_curves.loc[smoothing_curve_distances,
"scrubbing"] = scrub_smoothing_curve
del scrub_values, scrub_smoothing_curve
analysis_values.reset_index(inplace=True)
smoothing_curves.reset_index(inplace=True)
analysis_values.to_csv(
op.join(out_dir, "analysis_values.tsv.gz"),
sep="\t",
line_terminator="\n",
index=False,
)
smoothing_curves.to_csv(
op.join(out_dir, "smoothing_curves.tsv.gz"),
sep="\t",
line_terminator="\n",
index=False,
)
# Null distributions
LGR.info("Building null distributions with permutations")
qcrsfc_null_smoothing_curves, hl_null_smoothing_curves = analysis.other_null_distributions(
qc,
z_corr_mats,
distances,
window=window,
n_iters=n_iters,
n_jobs=n_jobs,
)
scrub_null_smoothing_curves = analysis.scrubbing_null_distribution(
qc,
ts_all,
distances,
qc_thresh,
edge_sorting_idx,
window=window,
n_iters=n_iters,
n_jobs=n_jobs,
)
np.savez_compressed(
op.join(out_dir, "null_smoothing_curves.npz"),
qcrsfc=qcrsfc_null_smoothing_curves,
highlow=hl_null_smoothing_curves,
scrubbing=scrub_null_smoothing_curves,
)
del qcrsfc_null_smoothing_curves, hl_null_smoothing_curves, scrub_null_smoothing_curves
plotting.plot_results(out_dir)
LGR.info("Workflow completed")
#
# We are going to use a single subject from the ADHD dataset.
from nilearn import datasets
adhd = datasets.fetch_adhd(n_subjects=1)
###############################################################################
# We store the paths to its functional image and the confounds file.
fmri_filename = adhd.func[0]
confounds_filename = adhd.confounds[0]
print('Functional image is {0},\nconfounds are {1}.'.format(
fmri_filename, confounds_filename))
###############################################################################
# We fetch the coordinates of Power atlas.
power = datasets.fetch_coords_power_2011()
print('Power atlas comes with {0}.'.format(power.keys()))
###############################################################################
# Compute within spheres averaged time-series
# -------------------------------------------
#
# We can compute the mean signal within **spheres** of a fixed radius around
# a sequence of (x, y, z) coordinates with the object
# :class:`nilearn.input_data.NiftiSpheresMasker`.
# So we collect the regions coordinates in a numpy array
import numpy as np
coords = np.vstack((power.rois['x'], power.rois['y'], power.rois['z'])).T
print('Stacked power coordinates in array of shape {0}.'.format(coords.shape))
# ------------------------------
#
# We are going to use a single subject from the ADHD dataset.
from nilearn import datasets
adhd = datasets.fetch_adhd(n_subjects=1)
###############################################################################
# We store the paths to its functional image and the confounds file.
fmri_filename = adhd.func[0]
confounds_filename = adhd.confounds[0]
print("Functional image is {0},\nconfounds are {1}.".format(fmri_filename, confounds_filename))
###############################################################################
# We fetch the coordinates of Power atlas.
power = datasets.fetch_coords_power_2011()
print("Power atlas comes with {0}.".format(power.keys()))
###############################################################################
# Compute within spheres averaged time-series
# -------------------------------------------
#
# We can compute the mean signal within **spheres** of a fixed radius around
# a sequence of (x, y, z) coordinates with the object
# :class:`nilearn.input_data.NiftiSpheresMasker`.
# So we collect the regions coordinates in a numpy array
import numpy as np
coords = np.vstack((power.rois["x"], power.rois["y"], power.rois["z"])).T
print("Stacked power coordinates in array of shape {0}.".format(coords.shape))
def make_functional_derivatives(population, workspace_dir, freesurfer_dir, derivatives_dir):
print '========================================================================================'
print ''
print ' Tourettome - 008. CREATING FUNCTIONAL FEATURES '
print ''
print '========================================================================================'
# global IO
sca_dir = mkdir_path(os.path.join(derivatives_dir, 'func_seed_correlation'))
gm_group_mask = os.path.join(derivatives_dir, 'func_centrality/GROUP_GM_FUNC_3mm.nii')
count = 0
for subject in population:
count +=1
print '###################################################################'
print 'Extracting functional derivatives for subject %s' % subject
# subject I/0
subject_dir = os.path.join(workspace_dir, subject)
for denoise_type in ['compcor', 'gsr','censor','gsr_censor']:
print 'Calculating Derivatives for denoise type:', denoise_type
sca_dir = mkdir_path(os.path.join(derivatives_dir, 'func_seed_correlation', denoise_type))
func_denoised = os.path.join(subject_dir, 'DENOISE', 'residuals_%s'%denoise_type, 'residual.nii.gz')
if os.path.isfile(func_denoised):
################################################################################################################
### 1- Seed-Based Correlation
################################################################################################################
print '1. Calculating SCA'
for seed_name in seeds:
if not os.path.isfile(os.path.join(sca_dir, seed_name, '%s_sca_z.nii.gz'%subject)):
print seed_name
seed_dir = mkdir_path(os.path.join(sca_dir, seed_name))
TR = nb.load(func_denoised).header['pixdim'][4]
# Extract seed timeseries
seed = seeds[seed_name]
masker_seed = NiftiLabelsMasker(labels_img=seed,
smoothing_fwhm=6, detrend = False, standardize=True,
low_pass = 0.1, high_pass = 0.01, t_r = TR,
memory='nilearn_cache', verbose=0,
)
timeseries_seed = masker_seed.fit_transform(func_denoised)
print 'seed_timeseries_shape', timeseries_seed.shape
# Extract brain timeseries
masker_brain = NiftiMasker(smoothing_fwhm=6, detrend=None, standardize=True,
low_pass=0.1, high_pass=0.01, t_r=TR,
memory='nilearn_cache', memory_level=1, verbose=0)
timeseries_brain = masker_brain.fit_transform(func_denoised)
print 'brain_timeseries_shape', timeseries_brain.shape
# Seed Based Correlation
# see Nilearn http://nilearn.github.io/auto_examples/03_connectivity/plot_seed_to_voxel_correlation.html#sphx-glr-auto-examples-03-connectivity-plot-seed-to-voxel-correlation-py
sca = np.dot(timeseries_brain.T, timeseries_seed) / timeseries_seed.shape[0]
sca_rz = np.arctanh(sca)
print("seed-based correlation R: min = %.3f; max = %.3f" % (sca.min(), sca.max()))
print("seed-based correlation R-to-Z : min = %.3f; max = %.3f" % (sca_rz.min(), sca_rz.max()))
# Save seed-to-brain correlation as a Nifti image
sca_img = masker_brain.inverse_transform(sca.T)
sca_img.to_filename(os.path.join(seed_dir, '%s_sca_z.nii.gz'%subject))
######### SKIP since already smoothed with nilearn
#smooth
# Smoothing kernel
# FWHM = 6
# sigma = FWHM / 2.35482004503
# os.chdir(seed_dir)
# os.system('fslmaths %s_sca_z -s %s %s_sca_z_fwhm6.nii.gz'%(subject, sigma, subject))
# skip the nilearn approach..... do it with freesurfer...
# Map seed-to-voxel onto surface
# sca_lh = surface.vol_to_surf(sca_img, fsaverage5['pial_left']).ravel()
# sca_rh = surface.vol_to_surf(sca_img, fsaverage5['pial_right']).ravel()
#
# # Save seed-to-vertex correlation as a txt file
# np.save(os.path.join(seed_dir, '%s_sca_z_fwhm6_lh.npy'%subject), sca_lh)
# np.save(os.path.join(seed_dir, '%s_sca_z_fwhm6_rh.npy'%subject), sca_rh)
####################
seed_dir = os.path.join(sca_dir, seed_name)
if not os.path.isfile(os.path.join(seed_dir, '%s_sca_z_fsaverage5_fwhm10_rh.mgh' % subject)):
os.chdir(seed_dir)
for hemi in ['lh', 'rh']:
# vol2surf
os.system('mri_vol2surf '
'--mov %s_sca_z.nii.gz '
'--regheader %s '
'--projfrac-avg 0.2 0.8 0.1 '
'--interp nearest '
'--hemi %s '
'--out %s_sca_z_%s.mgh'
%(subject, subject, hemi, subject, hemi))
#surf2surf
os.system('mri_surf2surf '
'--s %s '
'--sval %s_sca_z_%s.mgh '
'--hemi %s '
'--trgsubject fsaverage5 '
'--tval %s_sca_z_fsaverage5_fwhm00_%s.mgh'
% (subject, subject, hemi, hemi, subject, hemi))
os.system('mri_surf2surf '
'--s %s '
'--sval %s_sca_z_%s.mgh '
'--hemi %s '
'--trgsubject fsaverage5 '
'--fwhm-src 10 '
'--tval %s_sca_z_fsaverage5_fwhm10_%s.mgh'
%(subject, subject, hemi, hemi, subject, hemi))
os.system('rm -rf %s_sca_z_lh.mgh %s_sca_z_rh.mgh' %(subject,subject))
################################################################################################################
### 1- connectome
################################################################################################################
print '2. Calculating Power-264 connectome'
connectome_dir = mkdir_path(os.path.join(derivatives_dir, 'func_connectome', denoise_type))
if not os.path.isfile(os.path.join(connectome_dir, '%s_power264_tangent.npy'%subject)):
# get power-264 coordinates
atlas = datasets.fetch_coords_power_2011()
coords = np.vstack((atlas.rois['x'], atlas.rois['y'], atlas.rois['z'])).T
# Extract signals
spheres_masker = input_data.NiftiSpheresMasker(seeds=coords,
smoothing_fwhm=6,
radius=5.,
detrend=False,
standardize=True,
low_pass=0.1,
high_pass=0.01,
t_r=2.)
timeseries = spheres_masker.fit_transform(func_denoised)
print 'timsereies shape ', timeseries.shape
for cor_type in ['correlation', 'tangent']:
if not os.path.isfile(os.path.join(connectome_dir, '%s_power264_%s.npy' % (subject, cor_type))):
correlation_measure = connectome.ConnectivityMeasure(kind=cor_type)
cmat = correlation_measure.fit_transform([timeseries])[0]
np.save(os.path.join(connectome_dir, '%s_power264_%s.npy'%(subject,cor_type)), cmat)
else:
print 'Need denoising first'
##############################################################################
# uncomment this to open the plot in a web browser:
# view.open_in_browser()
##########################################################################
# Extract signals on spheres from an atlas
# ----------------------------------------
#
# Next, instead of supplying our own coordinates, we will use coordinates
# generated at the center of mass of regions from two different atlases.
# This time, we'll use a different correlation measure.
#
# First we fetch the coordinates of the Power atlas
power = datasets.fetch_coords_power_2011(legacy_format=False)
print('Power atlas comes with {0}.'.format(power.keys()))
#########################################################################
# .. note::
#
#
# You can retrieve the coordinates for any atlas, including atlases
# not included in nilearn, using
# :func:`nilearn.plotting.find_parcellation_cut_coords`.
###############################################################################
# Compute within spheres averaged time-series
# -------------------------------------------
#
# We collect the regions coordinates in a numpy array
def process_data(num=180):
# fp = "/home/jiaming/Desktop/ADNI_AV45PET/"
# # fp = "/Users/zhaoxuandong/Public/Dropbox (Partners HealthCare)/MImadrid/"
# # ep = "/Users/zhaoxuandong/Public/Dropbox (Partners HealthCare)/MImadrid/meta/metaData.xlsx"
print("loading data...")
if num == 120:
#
meta = np.load("AAL2.npy")
coo_dict = {}
# for i in range(1, 49):
# coo_dict[i] = np.where(meta_cort == i)
# for i in range(1, 22):
# coo_dict[i + 48] = np.where(meta_sub == i)
m = np.unique(meta)
for i in range(1, 121):
coo_dict[i] = np.where(meta == m[i])
# print(coo_dict[i])
vector_dict = []
for i in tqdm(range(1, num + 1)):
vector = []
for folder in ["NC", "EMCI", "LMCI", "AD"]:
for root, dirs, files in os.walk(folder):
for name in files:
if name[-1] != 'i':
continue
img = nib.load(os.path.join(root, name)).get_data()
mask = np.load('mask.npy')
img = img * mask
#img = (img - np.min(img))/(np.max(img) - np.min(img))
img = img[coo_dict[i]]
vector.append(np.average(img))
vector_dict.append(np.array(vector))
matrix_69 = np.array(vector_dict).transpose()
np.save("mat120", matrix_69)
print(matrix_69.shape) # 419 * 180
label = [0 for i in range(100)] + [1 for i in range(96)] + [
2 for i in range(131)
] + [3 for i in range(92)]
label = np.array(label)
np.save("label120", label)
elif num == 180:
#
meta = np.load("mmp.npy")
coo_dict = {}
# for i in range(1, 49):
# coo_dict[i] = np.where(meta_cort == i)
# for i in range(1, 22):
# coo_dict[i + 48] = np.where(meta_sub == i)
for i in range(1, 181):
coo_dict[i] = np.where(meta == i)
# print(coo_dict[i])
vector_dict = []
for i in tqdm(range(1, num + 1)):
vector = []
for folder in ["NC", "EMCI", "LMCI", "AD"]:
for root, dirs, files in os.walk(folder):
for name in files:
if name[-1] != 'i':
continue
img = nib.load(os.path.join(root, name)).get_data()
mask = np.load('mask.npy')
img = img * mask
#img = (img - np.min(img))/(np.max(img) - np.min(img))
img = img[coo_dict[i]]
vector.append(np.average(img))
vector_dict.append(np.array(vector))
matrix_69 = np.array(vector_dict).transpose()
np.save("mat180", matrix_69)
print(matrix_69.shape) # 419 * 180
label = [0 for i in range(100)] + [1 for i in range(96)] + [
2 for i in range(131)
] + [3 for i in range(92)]
label = np.array(label)
np.save("label180", label)
elif num == 69:
# fp = "/home/jiaming/Desktop/ADNI_AV45PET/"
meta_cort = nib.load(
"HarvardOxford-cort-maxprob-thr25-2mm.nii").get_data() # 48
meta_sub = nib.load(
"HarvardOxford-sub-maxprob-thr25-2mm.nii").get_data() # 21
coo_dict = {}
for i in range(1, 49):
coo_dict[i] = np.where(meta_cort == i)
for i in range(1, 22):
coo_dict[i + 48] = np.where(meta_sub == i)
vector_dict = []
for i in tqdm(range(1, num + 1)):
vector = []
for folder in ["NC", "EMCI", "LMCI", "AD"]:
for root, dirs, files in os.walk(folder):
for name in files:
if name[-1] != 'i':
continue
img = nib.load(os.path.join(root, name)).get_data()
mask = np.load('mask.npy')
img = img * mask
#img = (img - np.min(img))/(np.max(img) - np.min(img))
img = img[coo_dict[i]]
vector.append(np.average(img))
vector_dict.append(np.array(vector))
matrix_69 = np.array(vector_dict).transpose()
np.save("mat69", matrix_69)
print(matrix_69.shape) # 419 * 69
label = [0 for i in range(100)] + [1 for i in range(96)] + [
2 for i in range(131)
] + [3 for i in range(92)]
label = np.array(label)
np.save("label69", label)
elif num == 264:
from nilearn import datasets
adhd = datasets.fetch_adhd(n_subjects=1)
power = datasets.fetch_coords_power_2011()
mask = power.rois
mask = np.array([mask.x, mask.y, mask.z]).transpose()
mask[:, 0] = mask[:, 0] + 90
mask[:, 1] = mask[:, 1] + 130
mask[:, 2] = mask[:, 2] + 70
mask = mask / 2
mask = mask.astype(int)
print(mask.shape)
coo_dict = {}
assert mask.shape[0] == num
mask_264 = np.zeros([91, 109, 91])
for i in range(num):
xl = []
yl = []
zl = []
cx, cy, cz = mask[i]
#print(cx, cy, cz)
for x in [cx - 2, cx - 1, cx, cx + 1, cx + 2]:
for y in [cy - 2, cy - 1, cy, cy + 1, cy + 2]:
for z in [cz - 2, cz - 1, cz, cz + 1, cz + 2]:
if x >= 0 and y >= 0 and z >= 0 and x < 91 and y < 109 and z < 91:
if (x - cx)**2 + (y - cy)**2 + (z - cz)**2 < 10.25:
xl.append(x)
yl.append(y)
zl.append(z)
mask_264[x, y, z] = i + 1
coo_dict[i] = (np.array(xl), np.array(yl), np.array(zl))
#print(coo_dict)
print(np.unique(mask_264))
np.save("mask264", mask_264)
vector_dict = []
np.set_printoptions(suppress=True)
for i in tqdm(range(num)):
vector = []
for folder in ["NC", "EMCI", "LMCI", "AD"]:
for root, dirs, files in os.walk(folder):
for name in files:
if name[-1] != 'i':
continue
img = nib.load(os.path.join(root, name)).get_data()
mask = np.load('mask.npy')
img = img * mask
#img = (img - np.min(img))/(np.max(img) - np.min(img))
print(folder, img[[26, 57, 69]])
img = img[coo_dict[i]]
# print(img)
vector.append(np.average(img))
vector_dict.append(np.array(vector))
matrix_69 = np.array(vector_dict).transpose()
r, c = matrix_69.nonzero()
c_unique = np.unique(c)
matrix_69 = matrix_69[:, c_unique]
print(matrix_69.shape) # 419 * 264
np.save("mat264", matrix_69)
label = [0 for i in range(100)] + [1 for i in range(96)] + [
2 for i in range(131)
] + [3 for i in range(92)]
label = np.array(label)
np.save("label264", label)
else:
raise NotImplementedError
for pdata in PATH_TO_DATA:
subjects_path += glob.glob(pdata)
subjects_path = sorted(subjects_path)
PATH_TO_RESTING_STATE = 'session_1/rest_1/rest_res2standard.nii.gz'
PATH_TO_MOTION_CORRECTION = 'session_1/rest_1/rest_mc.1D'
# path to the atlases
ATLASES = [
fetch_atlas_msdl().maps,
fetch_atlas_basc_multiscale_2015().scale064,
fetch_atlas_basc_multiscale_2015().scale122,
fetch_atlas_basc_multiscale_2015().scale197,
fetch_atlas_harvard_oxford(atlas_name='cort-prob-2mm').maps,
fetch_atlas_craddock_2012().scorr_mean,
fetch_coords_power_2011()
]
ATLASES_DESCR = [
'msdl', 'basc064', 'basc122', 'basc197', 'harvard_oxford_cort_prob_2mm',
'craddock_scorr_mean', 'power_2011'
]
# load the list of patient to exclude
excluded_subjects = pd.read_csv(SUBJECTS_EXCLUDED,
dtype={'subject_id':
object})['subject_id'].tolist()
###############################################################################
# Build the list with all path
dataset = {'func': [], 'motion': [], 'subject_id': [], 'run': []}
for i, path_subject in enumerate(subjects_path):
'Ventral Attention', 'Limbic', 'Frontoparietal', 'Default'
]
ATLAS_3D = dict(
cortical=(
lambda: datasets.fetch_atlas_harvard_oxford('cort-maxprob-thr25-2mm')),
subcortical=(
lambda: datasets.fetch_atlas_harvard_oxford('sub-maxprob-thr25-2mm')),
yeo7=(lambda: dict(maps=datasets.fetch_atlas_yeo_2011()['thick_7'],
labels=YEO_LABELS)),
aal=(lambda: _fetch_aal()),
# brodmann=(lambda:
# datasets.fetch_atlas_talairach('ba'))
)
ATLAS_COORDS = dict(power=(lambda: datasets.fetch_coords_power_2011()), )
ATLAS_PROBABILISTIC = dict(msdl=(lambda: _fetch_msdl()))
def _fetch_aal():
""" The AAL atlas does not contain a background label.
To make the API consistent we fix it here.
"""
aal = datasets.fetch_atlas_aal()
aal['labels'] = ['Background'] + aal['labels']
return aal
def _fetch_msdl():
""" The AAL atlas does not contain a background label.
def run(files,
motpars,
out_dir='.',
n_iters=10000,
qc_thresh=0.2,
window=1000,
earl=False,
regress=False):
"""
Run scrubbing, high-low motion, and QCRSFC analyses.
Parameters
----------
files : (N,) list of nifti files
List of 4D (X x Y x Z x T) images in MNI space.
fds_analysis : (N,) list of array-like
List of 1D (T) numpy arrays with QC metric values per img (e.g., FD or
respiration).
out_dir : str
Output directory.
n_iters : int
Number of iterations to run to generate null distributions.
qc_thresh : float
Threshold for QC metric used in scrubbing analysis. Default is 0.2
(for FD).
window : int
Number of units (pairs of ROIs) to include when averaging to generate
smoothing curve.
Notes
-----
This function writes out several files to out_dir:
- all_sorted_distances.txt: Sorted distances between every pair of ROIs.
- smc_sorted_distances.txt: Sorted distances for smoothing curves. Does not
include duplicate distances or pairs of ROIs outside of smoothing curve
(first and last window/2 pairs).
- qcrsfc_analysis_values.txt: Results from QC:RSFC analysis. The QC:RSFC
value for each pair of ROIs, of the same size and in the same order as
all_sorted_distances.txt.
- qcrsfc_analysis_smoothing_curve.txt: Smoothing curve for QC:RSFC
analysis. Same size and order as smc_sorted_distances.txt.
- qcrsfc_analysis_null_smoothing_curves.txt: Null smoothing curves from
QC:RSFC analysis. Contains 2D array, where number of columns is same
size and order as smc_sorted_distances.txt and number of rows is number
of iterations for permutation analysis.
- highlow_analysis_values.txt: Results from high-low analysis. The delta r
value for each pair of ROIs, of the same size and in the same order as
all_sorted_distances.txt.
- highlow_analysis_smoothing_curve.txt: Smoothing curve for high-low
analysis. Same size and order as smc_sorted_distances.txt.
- highlow_analysis_null_smoothing_curves.txt: Null smoothing curves from
high-low analysis. Contains 2D array, where number of columns is same
size and order as smc_sorted_distances.txt and number of rows is number
of iterations for permutation analysis.
- scrubbing_analysis_values.txt: Results from scrubbing analysis. The
mean delta r value for each pair of ROIs, of the same size and in the
same order as all_sorted_distances.txt.
- scrubbing_analysis_smoothing_curve.txt: Smoothing curve for scrubbing
analysis. Same size and order as smc_sorted_distances.txt.
- scrubbing_analysis_null_smoothing_curves.txt: Null smoothing curves from
scrubbing analysis. Contains 2D array, where number of columns is same
size and order as smc_sorted_distances.txt and number of rows is number
of iterations for permutation analysis.
"""
# create logger with 'spam_application'
logger = logging.getLogger('ddmra')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler(op.join(out_dir, 'log.tsv'))
fh.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s\t%(name)-12s\t%(levelname)-8s\t%(message)s')
fh.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.info('Preallocating matrices')
assert len(files) == len(motpars)
n_subjects = len(files)
atlas = datasets.fetch_coords_power_2011()
coords = np.vstack((atlas.rois['x'], atlas.rois['y'], atlas.rois['z'])).T
n_rois = coords.shape[0]
mat_idx = np.triu_indices(n_rois, k=1)
dists = squareform(pdist(coords))
dists = dists[mat_idx]
sort_idx = dists.argsort()
all_sorted_dists = dists[sort_idx]
np.savetxt(op.join(out_dir, 'all_sorted_distances.txt'), all_sorted_dists)
un_idx = np.array([
np.where(all_sorted_dists == i)[0][0]
for i in np.unique(all_sorted_dists)
])
logger.info('Creating masker')
t_r = nib.load(files[0]).header.get_zooms()[-1]
spheres_masker = input_data.NiftiSpheresMasker(seeds=coords,
radius=5.,
t_r=t_r,
smoothing_fwhm=4.,
detrend=False,
standardize=False,
low_pass=None,
high_pass=None)
# if earl
motpars_analysis, fds_analysis = [], []
for motpars_ in motpars:
if earl:
logger.info('Filtering motion parameters')
motpars_filt, fd_filt = ef.filter_earl(motpars_, t_r)
motpars_analysis.append(motpars_filt)
fds_analysis.append(fd_filt)
else:
motpars_analysis.append(motpars_)
fds_analysis.append(ef.get_fd_power(motpars_, unit='rad'))
# prep for qcrsfc and high-low motion analyses
mean_qcs = np.array([np.mean(qc) for qc in fds_analysis])
raw_corr_mats = np.zeros((n_subjects, len(mat_idx[0])))
logger.info('Building correlation matrices')
# Get correlation matrices
ts_all = []
for i_sub in range(n_subjects):
img = nib.load(files[i_sub])
if regress:
raw_ts = spheres_masker.fit_transform(
img, confounds=motpars_analysis[i_sub]).T
else:
raw_ts = spheres_masker.fit_transform(img).T
ts_all.append(raw_ts)
raw_corrs = np.corrcoef(raw_ts)
raw_corrs = raw_corrs[mat_idx]
raw_corr_mats[i_sub, :] = raw_corrs
z_corr_mats = np.arctanh(raw_corr_mats)
del (raw_corrs, raw_ts, spheres_masker, atlas, coords)
# QC:RSFC r analysis
# For each pair of ROIs, correlate the z-transformed correlation
# coefficients across subjects with the subjects' mean QC (generally FD)
# values.
logger.info('Performing QC:RSFC analysis')
qcrsfc_rs = fast_pearson(z_corr_mats.T, mean_qcs)
qcrsfc_rs = qcrsfc_rs[sort_idx]
qcrsfc_smc = moving_average(qcrsfc_rs, window)
# Quick interlude to help reduce arrays
keep_idx = np.intersect1d(np.where(~np.isnan(qcrsfc_smc))[0], un_idx)
keep_sorted_dists = all_sorted_dists[keep_idx]
np.savetxt(op.join(out_dir, 'smc_sorted_distances.txt'), keep_sorted_dists)
# Now back to the QC:RSFC analysis
qcrsfc_smc = qcrsfc_smc[keep_idx]
np.savetxt(op.join(out_dir, 'qcrsfc_analysis_values.txt'), qcrsfc_rs)
np.savetxt(op.join(out_dir, 'qcrsfc_analysis_smoothing_curve.txt'),
qcrsfc_smc)
del qcrsfc_rs, qcrsfc_smc
# High-low motion analysis
# Split the sample using a median split of the QC metric (generally mean
# FD). Then, for each pair of ROIs, calculate the difference between the
# mean across correlation coefficients for the high motion minus the low
# motion groups.
logger.info('Performing high-low motion analysis')
hm_idx = mean_qcs >= np.median(mean_qcs)
lm_idx = mean_qcs < np.median(mean_qcs)
hm_mean_corr = np.mean(raw_corr_mats[hm_idx, :], axis=0)
lm_mean_corr = np.mean(raw_corr_mats[lm_idx, :], axis=0)
hl_corr_diff = hm_mean_corr - lm_mean_corr
hl_corr_diff = hl_corr_diff[sort_idx]
hl_smc = moving_average(hl_corr_diff, window)
hl_smc = hl_smc[keep_idx]
np.savetxt(op.join(out_dir, 'highlow_analysis_values.txt'), hl_corr_diff)
np.savetxt(op.join(out_dir, 'highlow_analysis_smoothing_curve.txt'),
hl_smc)
del hm_idx, lm_idx, hm_mean_corr, lm_mean_corr, hl_corr_diff, hl_smc
# Scrubbing analysis
logger.info('Performing scrubbing analysis')
mean_delta_r = scrubbing_analysis(ts_all,
fds_analysis,
n_rois,
qc_thresh,
perm=False)
mean_delta_r = mean_delta_r[sort_idx]
scrub_smc = moving_average(mean_delta_r, window)
scrub_smc = scrub_smc[keep_idx]
np.savetxt(op.join(out_dir, 'scrubbing_analysis_values.txt'), mean_delta_r)
np.savetxt(op.join(out_dir, 'scrubbing_analysis_smoothing_curve.txt'),
scrub_smc)
del mean_delta_r, scrub_smc
# Null distributions
logger.info('Building null distributions with permutations')
qcs_copy = [qc.copy() for qc in fds_analysis]
perm_scrub_smc = np.zeros((n_iters, len(keep_sorted_dists)))
perm_qcrsfc_smc = np.zeros((n_iters, len(keep_sorted_dists)))
perm_hl_smc = np.zeros((n_iters, len(keep_sorted_dists)))
for i in range(n_iters):
# Prep for QC:RSFC and high-low motion analyses
perm_mean_qcs = np.random.permutation(mean_qcs)
# QC:RSFC analysis
perm_qcrsfc_rs = fast_pearson(z_corr_mats.T, perm_mean_qcs)
perm_qcrsfc_rs = perm_qcrsfc_rs[sort_idx]
perm_qcrsfc_smc[i, :] = moving_average(perm_qcrsfc_rs,
window)[keep_idx]
# High-low analysis
perm_hm_idx = perm_mean_qcs >= np.median(perm_mean_qcs)
perm_lm_idx = perm_mean_qcs < np.median(perm_mean_qcs)
perm_hm_corr = np.mean(raw_corr_mats[perm_hm_idx, :], axis=0)
perm_lm_corr = np.mean(raw_corr_mats[perm_lm_idx, :], axis=0)
perm_hl_diff = perm_hm_corr - perm_lm_corr
perm_hl_diff = perm_hl_diff[sort_idx]
perm_hl_smc[i, :] = moving_average(perm_hl_diff, window)[keep_idx]
# Scrubbing analysis
perm_qcs = [np.random.permutation(perm_qc) for perm_qc in qcs_copy]
perm_mean_delta_r = scrubbing_analysis(ts_all,
perm_qcs,
n_rois,
qc_thresh,
perm=True)
perm_mean_delta_r = perm_mean_delta_r[sort_idx]
perm_scrub_smc[i, :] = moving_average(perm_mean_delta_r,
window)[keep_idx]
np.savetxt(op.join(out_dir, 'qcrsfc_analysis_null_smoothing_curves.txt'),
perm_qcrsfc_smc)
np.savetxt(op.join(out_dir, 'highlow_analysis_null_smoothing_curves.txt'),
perm_hl_smc)
np.savetxt(
op.join(out_dir, 'scrubbing_analysis_null_smoothing_curves.txt'),
perm_scrub_smc)
logger.info('Workflow completed')
logger.shutdown()