def map_beta(censor, coeff_df, power2011): # create a left censor censor_left = censor.copy() censor_left['beta'] = coeff_df # make symmetric censor df censor_right = censor_left[['connID', 'beta']].copy() censor_right['connID'] = censor_right['connID'].apply(reverse_connID) censor_LR = censor_left.append(censor_right, ignore_index=True) censor_LR = censor_LR[['connID', 'beta']] # merge to main roi df roi_df = get_ROI_df(power2011) roi_df = roi_df.merge(censor_LR, how="left", on=['connID']) roi_df['beta'] = roi_df['beta'].replace(np.nan, 0.0) # reformat beta matrix adj_vector = roi_df['beta'].values adj_beta = pd.DataFrame(vector2matrix(adj_vector), dtype='float') # grab center coordinates for atlas labels coor_vector = np.array([(0, 0, 0) for i in range(264)]) power = datasets.fetch_coords_power_2011() power_coords = np.vstack( (power.rois['x'], power.rois['y'], power.rois['z'])).T return adj_beta, power_coords
def run_analyses( files, qc, out_dir=".", confounds=None, n_iters=10000, n_jobs=1, qc_thresh=0.2, window=1000, ): """Run scrubbing, high-low motion, and QCRSFC analyses. Parameters ---------- files : (N,) list of nifti files List of 4D (X x Y x Z x T) images in MNI space. qc : (N,) list of array_like List of 1D (T) numpy arrays with QC metric values per img (e.g., FD or respiration). out_dir : str, optional Output directory. Default is current directory. confounds : None or (N,) list of array-like, optional List of 2D (T) numpy arrays with confounds per img. Default is None (no confounds are removed). n_iters : int, optional Number of iterations to run to generate null distributions. Default is 10000. n_jobs : int, optional The number of CPUs to use to do the computation. -1 means 'all CPUs'. Default is 1. qc_thresh : float, optional Threshold for QC metric used in scrubbing analysis. Default is 0.2 (for FD). window : int, optional Number of units (pairs of ROIs) to include when averaging to generate smoothing curve. Default is 1000. Notes ----- This function writes out several files to out_dir: - ``analysis_values.tsv.gz``: Raw analysis values for analyses. Has four columns: distance, qcrsfc, scrubbing, and highlow. - ``smoothing_curves.tsv.gz``: Smoothing curve information for analyses. Has four columns: distance, qcrsfc, scrubbing, and highlow. - ``null_smoothing_curves.npz``: Null smoothing curves from each analysis. Contains three 2D arrays, where number of columns is same size and order as distance column in ``smoothing_curves.tsv.gz`` and number of rows is number of iterations for permutation analysis. The three arrays' keys are 'qcrsfc', 'highlow', and 'scrubbing'. - ``[analysis]_analysis.png``: Figure for each analysis. """ makedirs(out_dir, exist_ok=True) # create LGR with 'spam_application' LGR.setLevel(logging.DEBUG) # create file handler which logs even debug messages fh = logging.FileHandler(op.join(out_dir, "log.tsv")) fh.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter( "%(asctime)s\t%(name)-12s\t%(levelname)-8s\t%(message)s") fh.setFormatter(formatter) # add the handlers to the LGR LGR.addHandler(fh) LGR.info("Preallocating matrices") n_subjects = len(files) # Load atlas and associated masker atlas = datasets.fetch_coords_power_2011() coords = np.vstack((atlas.rois["x"], atlas.rois["y"], atlas.rois["z"])).T n_rois = coords.shape[0] triu_idx = np.triu_indices(n_rois, k=1) distances = squareform(pdist(coords)) distances = distances[triu_idx] # Sorting index for distances edge_sorting_idx = distances.argsort() distances = distances[edge_sorting_idx] LGR.info("Creating masker") spheres_masker = input_data.NiftiSpheresMasker( seeds=coords, radius=5.0, t_r=None, smoothing_fwhm=4.0, detrend=False, standardize=False, low_pass=None, high_pass=None, ) # prep for qcrsfc and high-low motion analyses mean_qc = np.array([np.mean(subj_qc) for subj_qc in qc]) z_corr_mats = np.zeros((n_subjects, distances.size)) # Get correlation matrices ts_all = [] LGR.info("Building correlation matrices") if confounds: LGR.info("Regressing confounds out of data.") for i_subj in range(n_subjects): if confounds: raw_ts = spheres_masker.fit_transform( files[i_subj], confounds=confounds[i_subj]).T else: raw_ts = spheres_masker.fit_transform(files[i_subj]).T assert raw_ts.shape[0] == n_rois if np.any(np.isnan(raw_ts)): raise ValueError(f"Time series of {files[i_subj]} contains NaNs") roi_variances = np.var(raw_ts, axis=1) if any(roi_variances == 0): bad_rois = np.where(roi_variances == 0)[0] raise ValueError( f"ROI(s) {bad_rois} for {files[i_subj]} have variance of 0.") ts_all.append(raw_ts) raw_corrs = np.corrcoef(raw_ts) raw_corrs = raw_corrs[triu_idx] raw_corrs = raw_corrs[ edge_sorting_idx] # Sort from close to far ROI pairs z_corr_mats[i_subj, :] = np.arctanh(raw_corrs) del (raw_corrs, raw_ts, spheres_masker, atlas, coords) analysis_values = pd.DataFrame(columns=["qcrsfc", "highlow", "scrubbing"], index=distances) analysis_values.index.name = "distance" # QC:RSFC r analysis LGR.info("Performing QC:RSFC analysis") qcrsfc_values = analysis.qcrsfc_analysis(mean_qc, z_corr_mats) analysis_values["qcrsfc"] = qcrsfc_values qcrsfc_smoothing_curve = utils.moving_average(qcrsfc_values, window) qcrsfc_smoothing_curve, smoothing_curve_distances = utils.average_across_distances( qcrsfc_smoothing_curve, distances, ) # Quick interlude to create the smoothing_curves DataFrame smoothing_curves = pd.DataFrame( columns=["qcrsfc", "highlow", "scrubbing"], index=smoothing_curve_distances, ) smoothing_curves.index.name = "distance" smoothing_curves.loc[smoothing_curve_distances, "qcrsfc"] = qcrsfc_smoothing_curve del qcrsfc_values, qcrsfc_smoothing_curve # High-low motion analysis LGR.info("Performing high-low motion analysis") highlow_values = analysis.highlow_analysis(mean_qc, z_corr_mats) analysis_values["highlow"] = highlow_values hl_smoothing_curve = utils.moving_average(highlow_values, window) hl_smoothing_curve, smoothing_curve_distances = utils.average_across_distances( hl_smoothing_curve, distances, ) smoothing_curves.loc[smoothing_curve_distances, "highlow"] = hl_smoothing_curve del highlow_values, hl_smoothing_curve # Scrubbing analysis LGR.info("Performing scrubbing analysis") scrub_values = analysis.scrubbing_analysis(qc, ts_all, edge_sorting_idx, qc_thresh, perm=False) analysis_values["scrubbing"] = scrub_values scrub_smoothing_curve = utils.moving_average(scrub_values, window) scrub_smoothing_curve, smoothing_curve_distances = utils.average_across_distances( scrub_smoothing_curve, distances, ) smoothing_curves.loc[smoothing_curve_distances, "scrubbing"] = scrub_smoothing_curve del scrub_values, scrub_smoothing_curve analysis_values.reset_index(inplace=True) smoothing_curves.reset_index(inplace=True) analysis_values.to_csv( op.join(out_dir, "analysis_values.tsv.gz"), sep="\t", line_terminator="\n", index=False, ) smoothing_curves.to_csv( op.join(out_dir, "smoothing_curves.tsv.gz"), sep="\t", line_terminator="\n", index=False, ) # Null distributions LGR.info("Building null distributions with permutations") qcrsfc_null_smoothing_curves, hl_null_smoothing_curves = analysis.other_null_distributions( qc, z_corr_mats, distances, window=window, n_iters=n_iters, n_jobs=n_jobs, ) scrub_null_smoothing_curves = analysis.scrubbing_null_distribution( qc, ts_all, distances, qc_thresh, edge_sorting_idx, window=window, n_iters=n_iters, n_jobs=n_jobs, ) np.savez_compressed( op.join(out_dir, "null_smoothing_curves.npz"), qcrsfc=qcrsfc_null_smoothing_curves, highlow=hl_null_smoothing_curves, scrubbing=scrub_null_smoothing_curves, ) del qcrsfc_null_smoothing_curves, hl_null_smoothing_curves, scrub_null_smoothing_curves plotting.plot_results(out_dir) LGR.info("Workflow completed")
# # We are going to use a single subject from the ADHD dataset. from nilearn import datasets adhd = datasets.fetch_adhd(n_subjects=1) ############################################################################### # We store the paths to its functional image and the confounds file. fmri_filename = adhd.func[0] confounds_filename = adhd.confounds[0] print('Functional image is {0},\nconfounds are {1}.'.format( fmri_filename, confounds_filename)) ############################################################################### # We fetch the coordinates of Power atlas. power = datasets.fetch_coords_power_2011() print('Power atlas comes with {0}.'.format(power.keys())) ############################################################################### # Compute within spheres averaged time-series # ------------------------------------------- # # We can compute the mean signal within **spheres** of a fixed radius around # a sequence of (x, y, z) coordinates with the object # :class:`nilearn.input_data.NiftiSpheresMasker`. # So we collect the regions coordinates in a numpy array import numpy as np coords = np.vstack((power.rois['x'], power.rois['y'], power.rois['z'])).T print('Stacked power coordinates in array of shape {0}.'.format(coords.shape))
# ------------------------------ # # We are going to use a single subject from the ADHD dataset. from nilearn import datasets adhd = datasets.fetch_adhd(n_subjects=1) ############################################################################### # We store the paths to its functional image and the confounds file. fmri_filename = adhd.func[0] confounds_filename = adhd.confounds[0] print("Functional image is {0},\nconfounds are {1}.".format(fmri_filename, confounds_filename)) ############################################################################### # We fetch the coordinates of Power atlas. power = datasets.fetch_coords_power_2011() print("Power atlas comes with {0}.".format(power.keys())) ############################################################################### # Compute within spheres averaged time-series # ------------------------------------------- # # We can compute the mean signal within **spheres** of a fixed radius around # a sequence of (x, y, z) coordinates with the object # :class:`nilearn.input_data.NiftiSpheresMasker`. # So we collect the regions coordinates in a numpy array import numpy as np coords = np.vstack((power.rois["x"], power.rois["y"], power.rois["z"])).T print("Stacked power coordinates in array of shape {0}.".format(coords.shape))
def make_functional_derivatives(population, workspace_dir, freesurfer_dir, derivatives_dir): print '========================================================================================' print '' print ' Tourettome - 008. CREATING FUNCTIONAL FEATURES ' print '' print '========================================================================================' # global IO sca_dir = mkdir_path(os.path.join(derivatives_dir, 'func_seed_correlation')) gm_group_mask = os.path.join(derivatives_dir, 'func_centrality/GROUP_GM_FUNC_3mm.nii') count = 0 for subject in population: count +=1 print '###################################################################' print 'Extracting functional derivatives for subject %s' % subject # subject I/0 subject_dir = os.path.join(workspace_dir, subject) for denoise_type in ['compcor', 'gsr','censor','gsr_censor']: print 'Calculating Derivatives for denoise type:', denoise_type sca_dir = mkdir_path(os.path.join(derivatives_dir, 'func_seed_correlation', denoise_type)) func_denoised = os.path.join(subject_dir, 'DENOISE', 'residuals_%s'%denoise_type, 'residual.nii.gz') if os.path.isfile(func_denoised): ################################################################################################################ ### 1- Seed-Based Correlation ################################################################################################################ print '1. Calculating SCA' for seed_name in seeds: if not os.path.isfile(os.path.join(sca_dir, seed_name, '%s_sca_z.nii.gz'%subject)): print seed_name seed_dir = mkdir_path(os.path.join(sca_dir, seed_name)) TR = nb.load(func_denoised).header['pixdim'][4] # Extract seed timeseries seed = seeds[seed_name] masker_seed = NiftiLabelsMasker(labels_img=seed, smoothing_fwhm=6, detrend = False, standardize=True, low_pass = 0.1, high_pass = 0.01, t_r = TR, memory='nilearn_cache', verbose=0, ) timeseries_seed = masker_seed.fit_transform(func_denoised) print 'seed_timeseries_shape', timeseries_seed.shape # Extract brain timeseries masker_brain = NiftiMasker(smoothing_fwhm=6, detrend=None, standardize=True, low_pass=0.1, high_pass=0.01, t_r=TR, memory='nilearn_cache', memory_level=1, verbose=0) timeseries_brain = masker_brain.fit_transform(func_denoised) print 'brain_timeseries_shape', timeseries_brain.shape # Seed Based Correlation # see Nilearn http://nilearn.github.io/auto_examples/03_connectivity/plot_seed_to_voxel_correlation.html#sphx-glr-auto-examples-03-connectivity-plot-seed-to-voxel-correlation-py sca = np.dot(timeseries_brain.T, timeseries_seed) / timeseries_seed.shape[0] sca_rz = np.arctanh(sca) print("seed-based correlation R: min = %.3f; max = %.3f" % (sca.min(), sca.max())) print("seed-based correlation R-to-Z : min = %.3f; max = %.3f" % (sca_rz.min(), sca_rz.max())) # Save seed-to-brain correlation as a Nifti image sca_img = masker_brain.inverse_transform(sca.T) sca_img.to_filename(os.path.join(seed_dir, '%s_sca_z.nii.gz'%subject)) ######### SKIP since already smoothed with nilearn #smooth # Smoothing kernel # FWHM = 6 # sigma = FWHM / 2.35482004503 # os.chdir(seed_dir) # os.system('fslmaths %s_sca_z -s %s %s_sca_z_fwhm6.nii.gz'%(subject, sigma, subject)) # skip the nilearn approach..... do it with freesurfer... # Map seed-to-voxel onto surface # sca_lh = surface.vol_to_surf(sca_img, fsaverage5['pial_left']).ravel() # sca_rh = surface.vol_to_surf(sca_img, fsaverage5['pial_right']).ravel() # # # Save seed-to-vertex correlation as a txt file # np.save(os.path.join(seed_dir, '%s_sca_z_fwhm6_lh.npy'%subject), sca_lh) # np.save(os.path.join(seed_dir, '%s_sca_z_fwhm6_rh.npy'%subject), sca_rh) #################### seed_dir = os.path.join(sca_dir, seed_name) if not os.path.isfile(os.path.join(seed_dir, '%s_sca_z_fsaverage5_fwhm10_rh.mgh' % subject)): os.chdir(seed_dir) for hemi in ['lh', 'rh']: # vol2surf os.system('mri_vol2surf ' '--mov %s_sca_z.nii.gz ' '--regheader %s ' '--projfrac-avg 0.2 0.8 0.1 ' '--interp nearest ' '--hemi %s ' '--out %s_sca_z_%s.mgh' %(subject, subject, hemi, subject, hemi)) #surf2surf os.system('mri_surf2surf ' '--s %s ' '--sval %s_sca_z_%s.mgh ' '--hemi %s ' '--trgsubject fsaverage5 ' '--tval %s_sca_z_fsaverage5_fwhm00_%s.mgh' % (subject, subject, hemi, hemi, subject, hemi)) os.system('mri_surf2surf ' '--s %s ' '--sval %s_sca_z_%s.mgh ' '--hemi %s ' '--trgsubject fsaverage5 ' '--fwhm-src 10 ' '--tval %s_sca_z_fsaverage5_fwhm10_%s.mgh' %(subject, subject, hemi, hemi, subject, hemi)) os.system('rm -rf %s_sca_z_lh.mgh %s_sca_z_rh.mgh' %(subject,subject)) ################################################################################################################ ### 1- connectome ################################################################################################################ print '2. Calculating Power-264 connectome' connectome_dir = mkdir_path(os.path.join(derivatives_dir, 'func_connectome', denoise_type)) if not os.path.isfile(os.path.join(connectome_dir, '%s_power264_tangent.npy'%subject)): # get power-264 coordinates atlas = datasets.fetch_coords_power_2011() coords = np.vstack((atlas.rois['x'], atlas.rois['y'], atlas.rois['z'])).T # Extract signals spheres_masker = input_data.NiftiSpheresMasker(seeds=coords, smoothing_fwhm=6, radius=5., detrend=False, standardize=True, low_pass=0.1, high_pass=0.01, t_r=2.) timeseries = spheres_masker.fit_transform(func_denoised) print 'timsereies shape ', timeseries.shape for cor_type in ['correlation', 'tangent']: if not os.path.isfile(os.path.join(connectome_dir, '%s_power264_%s.npy' % (subject, cor_type))): correlation_measure = connectome.ConnectivityMeasure(kind=cor_type) cmat = correlation_measure.fit_transform([timeseries])[0] np.save(os.path.join(connectome_dir, '%s_power264_%s.npy'%(subject,cor_type)), cmat) else: print 'Need denoising first'
############################################################################## # uncomment this to open the plot in a web browser: # view.open_in_browser() ########################################################################## # Extract signals on spheres from an atlas # ---------------------------------------- # # Next, instead of supplying our own coordinates, we will use coordinates # generated at the center of mass of regions from two different atlases. # This time, we'll use a different correlation measure. # # First we fetch the coordinates of the Power atlas power = datasets.fetch_coords_power_2011(legacy_format=False) print('Power atlas comes with {0}.'.format(power.keys())) ######################################################################### # .. note:: # # # You can retrieve the coordinates for any atlas, including atlases # not included in nilearn, using # :func:`nilearn.plotting.find_parcellation_cut_coords`. ############################################################################### # Compute within spheres averaged time-series # ------------------------------------------- # # We collect the regions coordinates in a numpy array
def process_data(num=180): # fp = "/home/jiaming/Desktop/ADNI_AV45PET/" # # fp = "/Users/zhaoxuandong/Public/Dropbox (Partners HealthCare)/MImadrid/" # # ep = "/Users/zhaoxuandong/Public/Dropbox (Partners HealthCare)/MImadrid/meta/metaData.xlsx" print("loading data...") if num == 120: # meta = np.load("AAL2.npy") coo_dict = {} # for i in range(1, 49): # coo_dict[i] = np.where(meta_cort == i) # for i in range(1, 22): # coo_dict[i + 48] = np.where(meta_sub == i) m = np.unique(meta) for i in range(1, 121): coo_dict[i] = np.where(meta == m[i]) # print(coo_dict[i]) vector_dict = [] for i in tqdm(range(1, num + 1)): vector = [] for folder in ["NC", "EMCI", "LMCI", "AD"]: for root, dirs, files in os.walk(folder): for name in files: if name[-1] != 'i': continue img = nib.load(os.path.join(root, name)).get_data() mask = np.load('mask.npy') img = img * mask #img = (img - np.min(img))/(np.max(img) - np.min(img)) img = img[coo_dict[i]] vector.append(np.average(img)) vector_dict.append(np.array(vector)) matrix_69 = np.array(vector_dict).transpose() np.save("mat120", matrix_69) print(matrix_69.shape) # 419 * 180 label = [0 for i in range(100)] + [1 for i in range(96)] + [ 2 for i in range(131) ] + [3 for i in range(92)] label = np.array(label) np.save("label120", label) elif num == 180: # meta = np.load("mmp.npy") coo_dict = {} # for i in range(1, 49): # coo_dict[i] = np.where(meta_cort == i) # for i in range(1, 22): # coo_dict[i + 48] = np.where(meta_sub == i) for i in range(1, 181): coo_dict[i] = np.where(meta == i) # print(coo_dict[i]) vector_dict = [] for i in tqdm(range(1, num + 1)): vector = [] for folder in ["NC", "EMCI", "LMCI", "AD"]: for root, dirs, files in os.walk(folder): for name in files: if name[-1] != 'i': continue img = nib.load(os.path.join(root, name)).get_data() mask = np.load('mask.npy') img = img * mask #img = (img - np.min(img))/(np.max(img) - np.min(img)) img = img[coo_dict[i]] vector.append(np.average(img)) vector_dict.append(np.array(vector)) matrix_69 = np.array(vector_dict).transpose() np.save("mat180", matrix_69) print(matrix_69.shape) # 419 * 180 label = [0 for i in range(100)] + [1 for i in range(96)] + [ 2 for i in range(131) ] + [3 for i in range(92)] label = np.array(label) np.save("label180", label) elif num == 69: # fp = "/home/jiaming/Desktop/ADNI_AV45PET/" meta_cort = nib.load( "HarvardOxford-cort-maxprob-thr25-2mm.nii").get_data() # 48 meta_sub = nib.load( "HarvardOxford-sub-maxprob-thr25-2mm.nii").get_data() # 21 coo_dict = {} for i in range(1, 49): coo_dict[i] = np.where(meta_cort == i) for i in range(1, 22): coo_dict[i + 48] = np.where(meta_sub == i) vector_dict = [] for i in tqdm(range(1, num + 1)): vector = [] for folder in ["NC", "EMCI", "LMCI", "AD"]: for root, dirs, files in os.walk(folder): for name in files: if name[-1] != 'i': continue img = nib.load(os.path.join(root, name)).get_data() mask = np.load('mask.npy') img = img * mask #img = (img - np.min(img))/(np.max(img) - np.min(img)) img = img[coo_dict[i]] vector.append(np.average(img)) vector_dict.append(np.array(vector)) matrix_69 = np.array(vector_dict).transpose() np.save("mat69", matrix_69) print(matrix_69.shape) # 419 * 69 label = [0 for i in range(100)] + [1 for i in range(96)] + [ 2 for i in range(131) ] + [3 for i in range(92)] label = np.array(label) np.save("label69", label) elif num == 264: from nilearn import datasets adhd = datasets.fetch_adhd(n_subjects=1) power = datasets.fetch_coords_power_2011() mask = power.rois mask = np.array([mask.x, mask.y, mask.z]).transpose() mask[:, 0] = mask[:, 0] + 90 mask[:, 1] = mask[:, 1] + 130 mask[:, 2] = mask[:, 2] + 70 mask = mask / 2 mask = mask.astype(int) print(mask.shape) coo_dict = {} assert mask.shape[0] == num mask_264 = np.zeros([91, 109, 91]) for i in range(num): xl = [] yl = [] zl = [] cx, cy, cz = mask[i] #print(cx, cy, cz) for x in [cx - 2, cx - 1, cx, cx + 1, cx + 2]: for y in [cy - 2, cy - 1, cy, cy + 1, cy + 2]: for z in [cz - 2, cz - 1, cz, cz + 1, cz + 2]: if x >= 0 and y >= 0 and z >= 0 and x < 91 and y < 109 and z < 91: if (x - cx)**2 + (y - cy)**2 + (z - cz)**2 < 10.25: xl.append(x) yl.append(y) zl.append(z) mask_264[x, y, z] = i + 1 coo_dict[i] = (np.array(xl), np.array(yl), np.array(zl)) #print(coo_dict) print(np.unique(mask_264)) np.save("mask264", mask_264) vector_dict = [] np.set_printoptions(suppress=True) for i in tqdm(range(num)): vector = [] for folder in ["NC", "EMCI", "LMCI", "AD"]: for root, dirs, files in os.walk(folder): for name in files: if name[-1] != 'i': continue img = nib.load(os.path.join(root, name)).get_data() mask = np.load('mask.npy') img = img * mask #img = (img - np.min(img))/(np.max(img) - np.min(img)) print(folder, img[[26, 57, 69]]) img = img[coo_dict[i]] # print(img) vector.append(np.average(img)) vector_dict.append(np.array(vector)) matrix_69 = np.array(vector_dict).transpose() r, c = matrix_69.nonzero() c_unique = np.unique(c) matrix_69 = matrix_69[:, c_unique] print(matrix_69.shape) # 419 * 264 np.save("mat264", matrix_69) label = [0 for i in range(100)] + [1 for i in range(96)] + [ 2 for i in range(131) ] + [3 for i in range(92)] label = np.array(label) np.save("label264", label) else: raise NotImplementedError
for pdata in PATH_TO_DATA: subjects_path += glob.glob(pdata) subjects_path = sorted(subjects_path) PATH_TO_RESTING_STATE = 'session_1/rest_1/rest_res2standard.nii.gz' PATH_TO_MOTION_CORRECTION = 'session_1/rest_1/rest_mc.1D' # path to the atlases ATLASES = [ fetch_atlas_msdl().maps, fetch_atlas_basc_multiscale_2015().scale064, fetch_atlas_basc_multiscale_2015().scale122, fetch_atlas_basc_multiscale_2015().scale197, fetch_atlas_harvard_oxford(atlas_name='cort-prob-2mm').maps, fetch_atlas_craddock_2012().scorr_mean, fetch_coords_power_2011() ] ATLASES_DESCR = [ 'msdl', 'basc064', 'basc122', 'basc197', 'harvard_oxford_cort_prob_2mm', 'craddock_scorr_mean', 'power_2011' ] # load the list of patient to exclude excluded_subjects = pd.read_csv(SUBJECTS_EXCLUDED, dtype={'subject_id': object})['subject_id'].tolist() ############################################################################### # Build the list with all path dataset = {'func': [], 'motion': [], 'subject_id': [], 'run': []} for i, path_subject in enumerate(subjects_path):
'Ventral Attention', 'Limbic', 'Frontoparietal', 'Default' ] ATLAS_3D = dict( cortical=( lambda: datasets.fetch_atlas_harvard_oxford('cort-maxprob-thr25-2mm')), subcortical=( lambda: datasets.fetch_atlas_harvard_oxford('sub-maxprob-thr25-2mm')), yeo7=(lambda: dict(maps=datasets.fetch_atlas_yeo_2011()['thick_7'], labels=YEO_LABELS)), aal=(lambda: _fetch_aal()), # brodmann=(lambda: # datasets.fetch_atlas_talairach('ba')) ) ATLAS_COORDS = dict(power=(lambda: datasets.fetch_coords_power_2011()), ) ATLAS_PROBABILISTIC = dict(msdl=(lambda: _fetch_msdl())) def _fetch_aal(): """ The AAL atlas does not contain a background label. To make the API consistent we fix it here. """ aal = datasets.fetch_atlas_aal() aal['labels'] = ['Background'] + aal['labels'] return aal def _fetch_msdl(): """ The AAL atlas does not contain a background label.
def run(files, motpars, out_dir='.', n_iters=10000, qc_thresh=0.2, window=1000, earl=False, regress=False): """ Run scrubbing, high-low motion, and QCRSFC analyses. Parameters ---------- files : (N,) list of nifti files List of 4D (X x Y x Z x T) images in MNI space. fds_analysis : (N,) list of array-like List of 1D (T) numpy arrays with QC metric values per img (e.g., FD or respiration). out_dir : str Output directory. n_iters : int Number of iterations to run to generate null distributions. qc_thresh : float Threshold for QC metric used in scrubbing analysis. Default is 0.2 (for FD). window : int Number of units (pairs of ROIs) to include when averaging to generate smoothing curve. Notes ----- This function writes out several files to out_dir: - all_sorted_distances.txt: Sorted distances between every pair of ROIs. - smc_sorted_distances.txt: Sorted distances for smoothing curves. Does not include duplicate distances or pairs of ROIs outside of smoothing curve (first and last window/2 pairs). - qcrsfc_analysis_values.txt: Results from QC:RSFC analysis. The QC:RSFC value for each pair of ROIs, of the same size and in the same order as all_sorted_distances.txt. - qcrsfc_analysis_smoothing_curve.txt: Smoothing curve for QC:RSFC analysis. Same size and order as smc_sorted_distances.txt. - qcrsfc_analysis_null_smoothing_curves.txt: Null smoothing curves from QC:RSFC analysis. Contains 2D array, where number of columns is same size and order as smc_sorted_distances.txt and number of rows is number of iterations for permutation analysis. - highlow_analysis_values.txt: Results from high-low analysis. The delta r value for each pair of ROIs, of the same size and in the same order as all_sorted_distances.txt. - highlow_analysis_smoothing_curve.txt: Smoothing curve for high-low analysis. Same size and order as smc_sorted_distances.txt. - highlow_analysis_null_smoothing_curves.txt: Null smoothing curves from high-low analysis. Contains 2D array, where number of columns is same size and order as smc_sorted_distances.txt and number of rows is number of iterations for permutation analysis. - scrubbing_analysis_values.txt: Results from scrubbing analysis. The mean delta r value for each pair of ROIs, of the same size and in the same order as all_sorted_distances.txt. - scrubbing_analysis_smoothing_curve.txt: Smoothing curve for scrubbing analysis. Same size and order as smc_sorted_distances.txt. - scrubbing_analysis_null_smoothing_curves.txt: Null smoothing curves from scrubbing analysis. Contains 2D array, where number of columns is same size and order as smc_sorted_distances.txt and number of rows is number of iterations for permutation analysis. """ # create logger with 'spam_application' logger = logging.getLogger('ddmra') logger.setLevel(logging.DEBUG) # create file handler which logs even debug messages fh = logging.FileHandler(op.join(out_dir, 'log.tsv')) fh.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter( '%(asctime)s\t%(name)-12s\t%(levelname)-8s\t%(message)s') fh.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.info('Preallocating matrices') assert len(files) == len(motpars) n_subjects = len(files) atlas = datasets.fetch_coords_power_2011() coords = np.vstack((atlas.rois['x'], atlas.rois['y'], atlas.rois['z'])).T n_rois = coords.shape[0] mat_idx = np.triu_indices(n_rois, k=1) dists = squareform(pdist(coords)) dists = dists[mat_idx] sort_idx = dists.argsort() all_sorted_dists = dists[sort_idx] np.savetxt(op.join(out_dir, 'all_sorted_distances.txt'), all_sorted_dists) un_idx = np.array([ np.where(all_sorted_dists == i)[0][0] for i in np.unique(all_sorted_dists) ]) logger.info('Creating masker') t_r = nib.load(files[0]).header.get_zooms()[-1] spheres_masker = input_data.NiftiSpheresMasker(seeds=coords, radius=5., t_r=t_r, smoothing_fwhm=4., detrend=False, standardize=False, low_pass=None, high_pass=None) # if earl motpars_analysis, fds_analysis = [], [] for motpars_ in motpars: if earl: logger.info('Filtering motion parameters') motpars_filt, fd_filt = ef.filter_earl(motpars_, t_r) motpars_analysis.append(motpars_filt) fds_analysis.append(fd_filt) else: motpars_analysis.append(motpars_) fds_analysis.append(ef.get_fd_power(motpars_, unit='rad')) # prep for qcrsfc and high-low motion analyses mean_qcs = np.array([np.mean(qc) for qc in fds_analysis]) raw_corr_mats = np.zeros((n_subjects, len(mat_idx[0]))) logger.info('Building correlation matrices') # Get correlation matrices ts_all = [] for i_sub in range(n_subjects): img = nib.load(files[i_sub]) if regress: raw_ts = spheres_masker.fit_transform( img, confounds=motpars_analysis[i_sub]).T else: raw_ts = spheres_masker.fit_transform(img).T ts_all.append(raw_ts) raw_corrs = np.corrcoef(raw_ts) raw_corrs = raw_corrs[mat_idx] raw_corr_mats[i_sub, :] = raw_corrs z_corr_mats = np.arctanh(raw_corr_mats) del (raw_corrs, raw_ts, spheres_masker, atlas, coords) # QC:RSFC r analysis # For each pair of ROIs, correlate the z-transformed correlation # coefficients across subjects with the subjects' mean QC (generally FD) # values. logger.info('Performing QC:RSFC analysis') qcrsfc_rs = fast_pearson(z_corr_mats.T, mean_qcs) qcrsfc_rs = qcrsfc_rs[sort_idx] qcrsfc_smc = moving_average(qcrsfc_rs, window) # Quick interlude to help reduce arrays keep_idx = np.intersect1d(np.where(~np.isnan(qcrsfc_smc))[0], un_idx) keep_sorted_dists = all_sorted_dists[keep_idx] np.savetxt(op.join(out_dir, 'smc_sorted_distances.txt'), keep_sorted_dists) # Now back to the QC:RSFC analysis qcrsfc_smc = qcrsfc_smc[keep_idx] np.savetxt(op.join(out_dir, 'qcrsfc_analysis_values.txt'), qcrsfc_rs) np.savetxt(op.join(out_dir, 'qcrsfc_analysis_smoothing_curve.txt'), qcrsfc_smc) del qcrsfc_rs, qcrsfc_smc # High-low motion analysis # Split the sample using a median split of the QC metric (generally mean # FD). Then, for each pair of ROIs, calculate the difference between the # mean across correlation coefficients for the high motion minus the low # motion groups. logger.info('Performing high-low motion analysis') hm_idx = mean_qcs >= np.median(mean_qcs) lm_idx = mean_qcs < np.median(mean_qcs) hm_mean_corr = np.mean(raw_corr_mats[hm_idx, :], axis=0) lm_mean_corr = np.mean(raw_corr_mats[lm_idx, :], axis=0) hl_corr_diff = hm_mean_corr - lm_mean_corr hl_corr_diff = hl_corr_diff[sort_idx] hl_smc = moving_average(hl_corr_diff, window) hl_smc = hl_smc[keep_idx] np.savetxt(op.join(out_dir, 'highlow_analysis_values.txt'), hl_corr_diff) np.savetxt(op.join(out_dir, 'highlow_analysis_smoothing_curve.txt'), hl_smc) del hm_idx, lm_idx, hm_mean_corr, lm_mean_corr, hl_corr_diff, hl_smc # Scrubbing analysis logger.info('Performing scrubbing analysis') mean_delta_r = scrubbing_analysis(ts_all, fds_analysis, n_rois, qc_thresh, perm=False) mean_delta_r = mean_delta_r[sort_idx] scrub_smc = moving_average(mean_delta_r, window) scrub_smc = scrub_smc[keep_idx] np.savetxt(op.join(out_dir, 'scrubbing_analysis_values.txt'), mean_delta_r) np.savetxt(op.join(out_dir, 'scrubbing_analysis_smoothing_curve.txt'), scrub_smc) del mean_delta_r, scrub_smc # Null distributions logger.info('Building null distributions with permutations') qcs_copy = [qc.copy() for qc in fds_analysis] perm_scrub_smc = np.zeros((n_iters, len(keep_sorted_dists))) perm_qcrsfc_smc = np.zeros((n_iters, len(keep_sorted_dists))) perm_hl_smc = np.zeros((n_iters, len(keep_sorted_dists))) for i in range(n_iters): # Prep for QC:RSFC and high-low motion analyses perm_mean_qcs = np.random.permutation(mean_qcs) # QC:RSFC analysis perm_qcrsfc_rs = fast_pearson(z_corr_mats.T, perm_mean_qcs) perm_qcrsfc_rs = perm_qcrsfc_rs[sort_idx] perm_qcrsfc_smc[i, :] = moving_average(perm_qcrsfc_rs, window)[keep_idx] # High-low analysis perm_hm_idx = perm_mean_qcs >= np.median(perm_mean_qcs) perm_lm_idx = perm_mean_qcs < np.median(perm_mean_qcs) perm_hm_corr = np.mean(raw_corr_mats[perm_hm_idx, :], axis=0) perm_lm_corr = np.mean(raw_corr_mats[perm_lm_idx, :], axis=0) perm_hl_diff = perm_hm_corr - perm_lm_corr perm_hl_diff = perm_hl_diff[sort_idx] perm_hl_smc[i, :] = moving_average(perm_hl_diff, window)[keep_idx] # Scrubbing analysis perm_qcs = [np.random.permutation(perm_qc) for perm_qc in qcs_copy] perm_mean_delta_r = scrubbing_analysis(ts_all, perm_qcs, n_rois, qc_thresh, perm=True) perm_mean_delta_r = perm_mean_delta_r[sort_idx] perm_scrub_smc[i, :] = moving_average(perm_mean_delta_r, window)[keep_idx] np.savetxt(op.join(out_dir, 'qcrsfc_analysis_null_smoothing_curves.txt'), perm_qcrsfc_smc) np.savetxt(op.join(out_dir, 'highlow_analysis_null_smoothing_curves.txt'), perm_hl_smc) np.savetxt( op.join(out_dir, 'scrubbing_analysis_null_smoothing_curves.txt'), perm_scrub_smc) logger.info('Workflow completed') logger.shutdown()