def test_butterworth(): rand_gen = np.random.RandomState(0) n_features = 20000 n_samples = 100 sampling = 100 low_pass = 30 high_pass = 10 # Compare output for different options. # single timeseries data = rand_gen.randn(n_samples) data_original = data.copy() out_single = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False, save_memory=True) np.testing.assert_almost_equal(out_single, data) # multiple timeseries data = rand_gen.randn(n_samples, n_features) data[:, 0] = data_original # set first timeseries to previous data data_original = data.copy() out1 = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) # check that multiple- and single-timeseries filtering do the same thing. np.testing.assert_almost_equal(out1[:, 0], out_single) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False) np.testing.assert_almost_equal(out1, data) # Test nyquist frequency clipping, issue #482 out1 = nisignal.butterworth(data, sampling, low_pass=50., copy=True) out2 = nisignal.butterworth( data, sampling, low_pass=80., # Greater than nyq frequency copy=True) np.testing.assert_almost_equal(out1, out2)
def test_butterworth(): rand_gen = np.random.RandomState(0) n_features = 20000 n_samples = 100 sampling = 100 low_pass = 30 high_pass = 10 # Compare output for different options. # single timeseries data = rand_gen.randn(n_samples) data_original = data.copy() out_single = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False, save_memory=True) np.testing.assert_almost_equal(out_single, data) # multiple timeseries data = rand_gen.randn(n_samples, n_features) data[:, 0] = data_original # set first timeseries to previous data data_original = data.copy() out1 = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) # check that multiple- and single-timeseries filtering do the same thing. np.testing.assert_almost_equal(out1[:, 0], out_single) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False) np.testing.assert_almost_equal(out1, data) # Test nyquist frequency clipping, issue #482 out1 = nisignal.butterworth(data, sampling, low_pass=50., copy=True) out2 = nisignal.butterworth(data, sampling, low_pass=80., # Greater than nyq frequency copy=True) np.testing.assert_almost_equal(out1, out2)
def burgess_process(cifti_file, nifti_file, motion_file, hp, work_dir, out_file): """regress out motion from a cifti and filter""" cii = nib.load(cifti_file) data = cii.get_fdata() tr = cii.header.matrix.get_index_map(0).series_step motion = np.loadtxt(motion_file) confounds = motion_confounds(motion, hp, nifti_file, work_dir) data = data - confounds.dot(np.linalg.pinv(confounds, 1e-6).dot(data)) data = signal.butterworth(data, 1 / tr, high_pass=0.009, copy=True) out_cii = nib.cifti2.cifti2.Cifti2Image(data, cii.header, cii.nifti_header) out_cii.to_filename(out_file) return data
def filtered(PATH_GZ, Time_R): from nilearn.input_data import NiftiMasker from nilearn.signal import butterworth import os masker = NiftiMasker() signal = masker.fit_transform(PATH_GZ) X_filtered = butterworth(signals=signal, sampling_rate=1. / Time_R, high_pass=0.01, copy=True) fmri_filtered = masker.inverse_transform(X_filtered) OutFile = 'f' + PATH_GZ[PATH_GZ.rfind('/') + 1:] fmri_filtered.to_filename(OutFile) out_file = os.path.abspath(OutFile) return out_file
def test_butterworth(): rng = np.random.RandomState(42) n_features = 20000 n_samples = 100 sampling = 100 low_pass = 30 high_pass = 10 # Compare output for different options. # single timeseries data = rng.standard_normal(size=n_samples) data_original = data.copy() ''' May be only on py3.5: Bug in scipy 1.1.0 generates an unavoidable FutureWarning. (More info: https://github.com/scipy/scipy/issues/9086) The number of warnings generated is overwhelming TravisCI's log limit, causing it to fail tests. This hack prevents that and will be removed in future. ''' buggy_scipy = (LooseVersion(scipy.__version__) < LooseVersion('1.2') and LooseVersion(scipy.__version__) > LooseVersion('1.0')) if buggy_scipy: warnings.simplefilter('ignore') ''' END HACK ''' out_single = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False) np.testing.assert_almost_equal(out_single, data) np.testing.assert_(id(out_single) != id(data)) # multiple timeseries data = rng.standard_normal(size=(n_samples, n_features)) data[:, 0] = data_original # set first timeseries to previous data data_original = data.copy() out1 = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) np.testing.assert_(id(out1) != id(data_original)) # check that multiple- and single-timeseries filtering do the same thing. np.testing.assert_almost_equal(out1[:, 0], out_single) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False) np.testing.assert_almost_equal(out1, data) # Test nyquist frequency clipping, issue #482 out1 = nisignal.butterworth(data, sampling, low_pass=50., copy=True) out2 = nisignal.butterworth( data, sampling, low_pass=80., # Greater than nyq frequency copy=True) np.testing.assert_almost_equal(out1, out2) np.testing.assert_(id(out1) != id(out2))
def denoise(img_file, tsv_file, out_path, col_names=False, hp_filter=False, lp_filter=False, out_figure_path=False): nii_ext = '.nii.gz' FD_thr = [.5] sc_range = np.arange(-1, 3) constant = 'constant' # read in files img = load_niimg(img_file) # get file info img_name = os.path.basename(img.get_filename()) file_base = img_name[0:img_name.find('.')] save_img_file = pjoin(out_path, file_base + \ '_NR' + nii_ext) data = img.get_data() df_orig = pandas.read_csv(tsv_file, '\t', na_values='n/a') df = copy.deepcopy(df_orig) Ntrs = df.as_matrix().shape[0] print('# of TRs: ' + str(Ntrs)) assert (Ntrs == data.shape[len(data.shape) - 1]) # select columns to use as nuisance regressors if col_names: df = df[col_names] str_append = ' [SELECTED regressors in CSV]' else: col_names = df.columns.tolist() str_append = ' [ALL regressors in CSV]' # fill in missing nuisance values with mean for that variable for col in df.columns: if sum(df[col].isnull()) > 0: print('Filling in ' + str(sum(df[col].isnull())) + ' NaN value for ' + col) df[col] = df[col].fillna(np.mean(df[col])) print('# of Confound Regressors: ' + str(len(df.columns)) + str_append) # implement HP filter in regression TR = img.header.get_zooms()[-1] frame_times = np.arange(Ntrs) * TR if hp_filter: hp_filter = float(hp_filter) assert (hp_filter > 0) period_cutoff = 1. / hp_filter df = make_first_level_design_matrix(frame_times, period_cut=period_cutoff, add_regs=df.as_matrix(), add_reg_names=df.columns.tolist()) # fn adds intercept into dm hp_cols = [col for col in df.columns if 'drift' in col] print('# of High-pass Filter Regressors: ' + str(len(hp_cols))) else: # add in intercept column into data frame df[constant] = 1 print('No High-pass Filter Applied') dm = df.as_matrix() # prep data data = np.reshape(data, (-1, Ntrs)) data_mean = np.mean(data, axis=1) Nvox = len(data_mean) # setup and run regression model = regression.OLSModel(dm) results = model.fit(data.T) if not hp_filter: results_orig_resid = copy.deepcopy(results.resid) # save for rsquared computation # apply low-pass filter if lp_filter: # input to butterworth fn is time x voxels low_pass = float(lp_filter) Fs = 1. / TR if low_pass >= Fs / 2: raise ValueError('Low pass filter cutoff if too close to the Nyquist frequency (%s)' % (Fs / 2)) temp_img_file = pjoin(out_path, file_base + \ '_temp' + nii_ext) temp_img = nb.Nifti1Image(np.reshape(results.resid.T + np.reshape(data_mean, (Nvox, 1)), img.shape).astype('float32'), img.affine, header=img.header) temp_img.to_filename(temp_img_file) results.resid = butterworth(results.resid, sampling_rate=Fs, low_pass=low_pass, high_pass=None) print('Low-pass Filter Applied: < ' + str(low_pass) + ' Hz') # add mean back into data clean_data = results.resid.T + np.reshape(data_mean, (Nvox, 1)) # add mean back into residuals # save out new data file print('Saving output file...') clean_data = np.reshape(clean_data, img.shape).astype('float32') new_img = nb.Nifti1Image(clean_data, img.affine, header=img.header) new_img.to_filename(save_img_file) ######### generate Rsquared map for confounds only if hp_filter: # first remove low-frequency information from data hp_cols.append(constant) model_first = regression.OLSModel(df[hp_cols].as_matrix()) results_first = model_first.fit(data.T) results_first_resid = copy.deepcopy(results_first.resid) del results_first, model_first # compute sst - borrowed from matlab sst = np.square(np.linalg.norm(results_first_resid - np.mean(results_first_resid, axis=0), axis=0)) # now regress out 'true' confounds to estimate their Rsquared nr_cols = [col for col in df.columns if 'drift' not in col] model_second = regression.OLSModel(df[nr_cols].as_matrix()) results_second = model_second.fit(results_first_resid) # compute sse - borrowed from matlab sse = np.square(np.linalg.norm(results_second.resid, axis=0)) del results_second, model_second, results_first_resid elif not hp_filter: # compute sst - borrowed from matlab sst = np.square(np.linalg.norm(data.T - np.mean(data.T, axis=0), axis=0)) # compute sse - borrowed from matlab sse = np.square(np.linalg.norm(results_orig_resid, axis=0)) del results_orig_resid # compute rsquared of nuisance regressors zero_idx = scipy.logical_and(sst == 0, sse == 0) sse[zero_idx] = 1 sst[zero_idx] = 1 # would be NaNs - become rsquared = 0 rsquare = 1 - np.true_divide(sse, sst) rsquare[np.isnan(rsquare)] = 0 ######### Visualizing DM & outputs fontsize = 12 fontsize_title = 14 def_img_size = 8 if not out_figure_path: out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures' if not os.path.isdir(out_figure_path): os.mkdir(out_figure_path) png_append = '_' + img_name[0:img_name.find('.')] + '.png' print('Output directory: ' + out_figure_path) # DM corr matrix cm = df[df.columns[0:-1]].corr() curr_sz = copy.deepcopy(def_img_size) if cm.shape[0] > def_img_size: curr_sz = curr_sz + ((cm.shape[0] - curr_sz) * .3) mtx_scale = curr_sz * 100 mask = np.zeros_like(cm, dtype=np.bool) mask[np.triu_indices_from(mask)] = True fig, ax = plt.subplots(figsize=(curr_sz, curr_sz)) cmap = sns.diverging_palette(220, 10, as_cmap=True) sns.heatmap(cm, mask=mask, cmap=cmap, center=0, vmax=cm[cm < 1].max().max(), vmin=cm[cm < 1].min().min(), square=True, linewidths=.5, cbar_kws={"shrink": .6}) ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize) ax.set_yticklabels(cm.columns.tolist(), rotation=-30, va='bottom', fontsize=fontsize) ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title) plt.tight_layout() file_corr_matrix = 'Corr_matrix_regressors' + png_append fig.savefig(pjoin(out_figure_path, file_corr_matrix)) plt.close(fig) del fig, ax # DM of Nuisance Regressors (all) tr_label = 'TR (Volume #)' fig, ax = plt.subplots(figsize=(curr_sz - 4.1, def_img_size)) x_scale_html = ((curr_sz - 4.1) / def_img_size) * 890 reporting.plot_design_matrix(df, ax=ax) ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title) ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize) ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize) ax.set_ylabel(tr_label, fontsize=fontsize) plt.tight_layout() file_design_matrix = 'Design_matrix' + png_append fig.savefig(pjoin(out_figure_path, file_design_matrix)) plt.close(fig) del fig, ax # FD timeseries plot FD = 'FD' poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd'] fd_idx = [df_orig.columns.__contains__(i) for i in poss_names] if np.sum(fd_idx) > 0: FD_name = poss_names[fd_idx == True] if sum(df_orig[FD_name].isnull()) > 0: df_orig[FD_name] = df_orig[FD_name].fillna(np.mean(df_orig[FD_name])) y = df_orig[FD_name].as_matrix() Nremove = [] sc_idx = [] for thr_idx, thr in enumerate(FD_thr): idx = y >= thr sc_idx.append(copy.deepcopy(idx)) for iidx in np.where(idx)[0]: for buffer in sc_range: curr_idx = iidx + buffer if curr_idx >= 0 and curr_idx <= len(idx): sc_idx[thr_idx][curr_idx] = True Nremove.append(np.sum(sc_idx[thr_idx])) Nplots = len(FD_thr) sns.set(font_scale=1.5) sns.set_style('ticks') fig, axes = plt.subplots(Nplots, 1, figsize=(def_img_size * 1.5, def_img_size / 2), squeeze=False) sns.despine() bound = .4 fd_mean = np.mean(y) for curr in np.arange(0, Nplots): axes[curr, 0].plot(y) axes[curr, 0].plot((-bound, Ntrs + bound), FD_thr[curr] * np.ones((1, 2))[0], '--', color='black') axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20) if Nremove[curr] > 0: info = scipy.ndimage.measurements.label(sc_idx[curr]) for cluster in np.arange(1, info[1] + 1): temp = np.where(info[0] == cluster)[0] axes[curr, 0].axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red') axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')') axes[curr, 0].set_title(FD + ': ' + str(100 * Nremove[curr] / Ntrs)[0:4] + '% of scan (' + str(Nremove[curr]) + ' volumes) would be scrubbed (FD thr.= ' + str(FD_thr[curr]) + ')') plt.text(Ntrs + 1, FD_thr[curr] - .01, FD + ' = ' + str(FD_thr[curr]), fontsize=fontsize) plt.text(Ntrs, fd_mean - .01, 'avg = ' + str(fd_mean), fontsize=fontsize) axes[curr, 0].set_xlim((-bound, Ntrs + 8)) plt.tight_layout() axes[curr, 0].set_xlabel(tr_label) file_fd_plot = FD + '_timeseries' + png_append fig.savefig(pjoin(out_figure_path, file_fd_plot)) plt.close(fig) del fig, axes print(FD + ' timeseries plot saved') else: print(FD + ' not found: ' + FD + ' timeseries not plotted') file_fd_plot = None # Carpet and DVARS plots - before & after nuisance regression # need to create mask file to input to DVARS function mask_file = pjoin(out_figure_path, 'mask_temp.nii.gz') nifti_masker = NiftiMasker(mask_strategy='epi', standardize=False) nifti_masker.fit(img) nifti_masker.mask_img_.to_filename(mask_file) # create 2 or 3 carpet plots, depending on if LP filter is also applied Ncarpet = 2 total_sz = int(16) carpet_scale = 840 y_labels = ['Input (voxels)', 'Output \'cleaned\''] imgs = [img, new_img] img_files = [img_file, save_img_file] color = ['red', 'salmon'] labels = ['input', 'cleaned'] if lp_filter: Ncarpet = 3 total_sz = int(20) carpet_scale = carpet_scale * (9/8) y_labels = ['Input', 'Clean Pre-LP', 'Clean LP'] imgs.insert(1, temp_img) img_files.insert(1, temp_img_file) color.insert(1, 'firebrick') labels.insert(1, 'clean pre-LP') labels[-1] = 'clean LP' dvars = [] print('Computing dvars...') for in_file in img_files: temp = nac.compute_dvars(in_file=in_file, in_mask=mask_file)[1] dvars.append(np.hstack((temp.mean(), temp))) del temp small_sz = 2 fig = plt.figure(figsize=(def_img_size * 1.5, def_img_size + ((Ncarpet - 2) * 1))) row_used = 0 if np.sum(fd_idx) > 0: # if FD data is available row_used = row_used + small_sz ax0 = plt.subplot2grid((total_sz, 1), (0, 0), rowspan=small_sz) ax0.plot(y) ax0.scatter(np.arange(0, Ntrs), y, s=10) curr = 0 if Nremove[curr] > 0: info = scipy.ndimage.measurements.label(sc_idx[curr]) for cluster in np.arange(1, info[1] + 1): temp = np.where(info[0] == cluster)[0] ax0.axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red') ax0.set_ylabel(FD) for side in ["top", "right", "bottom"]: ax0.spines[side].set_color('none') ax0.spines[side].set_visible(False) ax0.set_xticks([]) ax0.set_xlim((-.5, Ntrs - .5)) ax0.spines["left"].set_position(('outward', 10)) ax_d = plt.subplot2grid((total_sz, 1), (row_used, 0), rowspan=small_sz) for iplot in np.arange(len(dvars)): ax_d.plot(dvars[iplot], color=color[iplot], label=labels[iplot]) ax_d.set_ylabel('DVARS') for side in ["top", "right", "bottom"]: ax_d.spines[side].set_color('none') ax_d.spines[side].set_visible(False) ax_d.set_xticks([]) ax_d.set_xlim((-.5, Ntrs - .5)) ax_d.spines["left"].set_position(('outward', 10)) ax_d.legend(fontsize=fontsize - 2) row_used = row_used + small_sz st = 0 carpet_each = int((total_sz - row_used) / Ncarpet) for idx, img_curr in enumerate(imgs): ax_curr = plt.subplot2grid((total_sz, 1), (row_used + st, 0), rowspan=carpet_each) fig = plotting.plot_carpet(img_curr, figure=fig, axes=ax_curr) ax_curr.set_ylabel(y_labels[idx]) for side in ["bottom", "left"]: ax_curr.spines[side].set_position(('outward', 10)) if idx < len(imgs)-1: ax_curr.spines["bottom"].set_visible(False) ax_curr.set_xticklabels('') ax_curr.set_xlabel('') st = st + carpet_each file_carpet_plot = 'Carpet_plots' + png_append fig.savefig(pjoin(out_figure_path, file_carpet_plot)) plt.close() del fig, ax0, ax_curr, ax_d, dvars os.remove(mask_file) print('Carpet/DVARS plots saved') if lp_filter: os.remove(temp_img_file) del temp_img # Display T-stat maps for nuisance regressors # create mean img img_size = (img.shape[0], img.shape[1], img.shape[2]) mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine) mx = [] for idx, col in enumerate(df.columns): if not 'drift' in col and not constant in col: con_vector = np.zeros((1, df.shape[1])) con_vector[0, idx] = 1 con = results.Tcontrast(con_vector) mx.append(np.max(np.absolute([con.t.min(), con.t.max()]))) mx = .8 * np.max(mx) t_png = 'Tstat_' file_tstat = [] for idx, col in enumerate(df.columns): if not 'drift' in col and not constant in col: con_vector = np.zeros((1, df.shape[1])) con_vector[0, idx] = 1 con = results.Tcontrast(con_vector) m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine) title_str = col + ' Tstat' fig = plotting.plot_stat_map(m_img, mean_img, threshold=3, colorbar=True, display_mode='z', vmax=mx, title=title_str, cut_coords=7) file_temp = t_png + col + png_append fig.savefig(pjoin(out_figure_path, file_temp)) file_tstat.append({'name': col, 'file': file_temp}) plt.close() del fig, file_temp print(title_str + ' map saved') # Display R-sq map for nuisance regressors m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine) title_str = 'Nuisance Rsq' mx = .95 * rsquare.max() fig = plotting.plot_stat_map(m_img, mean_img, threshold=.2, colorbar=True, display_mode='z', vmax=mx, title=title_str, cut_coords=7) file_rsq_map = 'Rsquared' + png_append fig.savefig(pjoin(out_figure_path, file_rsq_map)) plt.close() del fig print(title_str + ' map saved') ######### html report templateLoader = jinja2.FileSystemLoader(searchpath="/") templateEnv = jinja2.Environment(loader=templateLoader) templateVars = {"img_file": img_file, "save_img_file": save_img_file, "Ntrs": Ntrs, "tsv_file": tsv_file, "col_names": col_names, "hp_filter": hp_filter, "lp_filter": lp_filter, "file_design_matrix": file_design_matrix, "file_corr_matrix": file_corr_matrix, "file_fd_plot": file_fd_plot, "file_rsq_map": file_rsq_map, "file_tstat": file_tstat, "x_scale": x_scale_html, "mtx_scale": mtx_scale, "file_carpet_plot": file_carpet_plot, "carpet_scale": carpet_scale } TEMPLATE_FILE = pjoin(os.getcwd(), "report_template.html") template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(templateVars) html_file = pjoin(out_figure_path, img_name[0:img_name.find('.')] + '.html') with open(html_file, "w") as f: f.write(outputText) print('') print('HTML report: ' + html_file) return new_img
if (hdr_input.nSamples - 1) < endsample: print "Error: buffer reset detected" raise SystemExit endsample = hdr_input.nSamples - 1 if endsample < window: continue begsample = endsample - window + 1 D = ftc.getData([begsample, endsample]) D = D[:, chanindx] if low_pass or high_pass: D = signal.butterworth(D, hdr_input.fSample, low_pass=low_pass, high_pass=high_pass, order=order) rms = [] for i in range(0, len(chanindx)): rms.append(0) for i, chanvec in enumerate(D.transpose()): for chanval in chanvec: rms[i] += chanval * chanval rms[i] = math.sqrt(rms[i]) if debug > 1: print rms
def _run_interface(self, runtime): import os import numpy as np import pandas as pd import SimpleITK as sitk import nilearn.image import nibabel as nb from nilearn.signal import butterworth from scipy.signal import detrend from rabies.confound_correction_pkg.utils import recover_3D,recover_4D,temporal_censoring,lombscargle_fill, exec_ICA_AROMA,butterworth from rabies.analysis_pkg.analysis_functions import closed_form ### set null returns in case the workflow is interrupted empty_img = sitk.GetImageFromArray(np.empty([1,1])) empty_file = os.path.abspath('empty.nii.gz') sitk.WriteImage(empty_img, empty_file) setattr(self, 'cleaned_path', empty_file) setattr(self, 'VE_file_path', empty_file) setattr(self, 'STD_file_path', empty_file) setattr(self, 'CR_STD_file_path', empty_file) setattr(self, 'frame_mask_file', empty_file) setattr(self, 'data_dict', empty_file) setattr(self, 'aroma_out', empty_file) ### bold_file = self.inputs.bold_file brain_mask_file = self.inputs.brain_mask_file CSF_mask_file = self.inputs.CSF_mask_file data_dict = self.inputs.data_dict cr_opts = self.inputs.cr_opts FD_trace=data_dict['FD_trace'] confounds_array=data_dict['confounds_array'] confounds_file=data_dict['confounds_csv'] time_range=data_dict['time_range'] confounds_6rigid_array=data_dict['confounds_6rigid_array'] cr_out = os.getcwd() import pathlib # Better path manipulation filename_split = pathlib.Path(bold_file).name.rsplit(".nii") brain_mask = sitk.GetArrayFromImage(sitk.ReadImage(brain_mask_file, sitk.sitkFloat32)) volume_indices = brain_mask.astype(bool) data_img = sitk.ReadImage(bold_file, sitk.sitkFloat32) data_array = sitk.GetArrayFromImage(data_img) num_volumes = data_array.shape[0] timeseries = np.zeros([num_volumes, volume_indices.sum()]) for i in range(num_volumes): timeseries[i, :] = (data_array[i, :, :, :])[volume_indices] timeseries = timeseries[time_range,:] if cr_opts.TR=='auto': TR = float(data_img.GetSpacing()[3]) else: TR = float(cr_opts.TR) ''' #1 - Compute and apply frame censoring mask (from FD and/or DVARS thresholds) ''' frame_mask,FD_trace,DVARS = temporal_censoring(timeseries, FD_trace, cr_opts.FD_censoring, cr_opts.FD_threshold, cr_opts.DVARS_censoring, cr_opts.minimum_timepoint) if frame_mask is None: return runtime timeseries = timeseries[frame_mask] confounds_array = confounds_array[frame_mask] ''' #2 - Linear detrending of fMRI timeseries and nuisance regressors ''' # apply simple detrending, after censoring timeseries = detrend(timeseries,axis=0) confounds_array = detrend(confounds_array,axis=0) # apply detrending to the confounds too ''' #3 - Apply ICA-AROMA. ''' if cr_opts.run_aroma: # write intermediary output files for timeseries and 6 rigid body parameters timeseries_3d = recover_4D(brain_mask_file, timeseries, bold_file) inFile = f'{cr_out}/{filename_split[0]}_aroma_input.nii.gz' sitk.WriteImage(timeseries_3d, inFile) confounds_6rigid_array=confounds_6rigid_array[frame_mask,:] confounds_6rigid_array = detrend(confounds_6rigid_array,axis=0) # apply detrending to the confounds too df = pd.DataFrame(confounds_6rigid_array) df.columns = ['mov1', 'mov2', 'mov3', 'rot1', 'rot2', 'rot3'] mc_file = f'{cr_out}/{filename_split[0]}_aroma_input.csv' df.to_csv(mc_file) cleaned_file, aroma_out = exec_ICA_AROMA(inFile, mc_file, brain_mask_file, CSF_mask_file, TR, cr_opts.aroma_dim, random_seed=cr_opts.aroma_random_seed) # if AROMA failed, returns empty outputs if cleaned_file is None: return runtime setattr(self, 'aroma_out', aroma_out) data_img = sitk.ReadImage(cleaned_file, sitk.sitkFloat32) data_array = sitk.GetArrayFromImage(data_img) num_volumes = data_array.shape[0] timeseries = np.zeros([num_volumes, volume_indices.sum()]) for i in range(num_volumes): timeseries[i, :] = (data_array[i, :, :, :])[volume_indices] if (not cr_opts.highpass is None) or (not cr_opts.lowpass is None): ''' #4 - If frequency filtering and frame censoring are applied, simulate data in censored timepoints using the Lomb-Scargle periodogram, as suggested in Power et al. (2014, Neuroimage), for both the fMRI timeseries and nuisance regressors prior to filtering. ''' timeseries_filled = lombscargle_fill(x=timeseries,time_step=TR,time_mask=frame_mask) confounds_filled = lombscargle_fill(x=confounds_array,time_step=TR,time_mask=frame_mask) ''' #5 - As recommended in Lindquist et al. (2019, Human brain mapping), make the nuisance regressors orthogonal to the temporal filter. ''' confounds_filtered = butterworth(confounds_filled, TR=TR, high_pass=cr_opts.highpass, low_pass=cr_opts.lowpass) ''' #6 - Apply highpass and/or lowpass filtering on the fMRI timeseries (with simulated timepoints). ''' timeseries_filtered = butterworth(timeseries_filled, TR=TR, high_pass=cr_opts.highpass, low_pass=cr_opts.lowpass) # correct for edge effects of the filters num_cut = int(cr_opts.edge_cutoff/TR) if len(frame_mask)<2*num_cut: raise ValueError(f"The timeseries are too short to remove {cr_opts.edge_cutoff}sec of data at each edge.") if not num_cut==0: frame_mask[:num_cut]=0 frame_mask[-num_cut:]=0 ''' #7 - Re-apply the frame censoring mask onto filtered fMRI timeseries and nuisance regressors, taking out the simulated timepoints. Edge artefacts from frequency filtering can also be removed as recommended in Power et al. (2014, Neuroimage). ''' # re-apply the masks to take out simulated data points, and take off the edges timeseries = timeseries_filtered[frame_mask] confounds_array = confounds_filtered[frame_mask] if frame_mask.sum()<int(cr_opts.minimum_timepoint): from nipype import logging log = logging.getLogger('nipype.workflow') log.warning(f"CONFOUND CORRECTION LEFT LESS THAN {str(cr_opts.minimum_timepoint)} VOLUMES. THIS SCAN WILL BE REMOVED FROM FURTHER PROCESSING.") return runtime ''' #8 - Apply confound regression using the selected nuisance regressors. ''' # voxels that have a NaN value are set to 0 nan_voxels = np.isnan(timeseries).sum(axis=0)>1 timeseries[:,nan_voxels] = 0 # estimate the VE from the CR selection, or 6 rigid motion parameters if no CR is applied X=confounds_array Y=timeseries try: predicted = X.dot(closed_form(X,Y)) res = Y-predicted except: from nipype import logging log = logging.getLogger('nipype.workflow') log.warning("SINGULAR MATRIX ERROR DURING CONFOUND REGRESSION. THIS SCAN WILL BE REMOVED FROM FURTHER PROCESSING.") empty_img = sitk.GetImageFromArray(np.empty([1,1])) empty_file = os.path.abspath('empty.nii.gz') sitk.WriteImage(empty_img, empty_file) return runtime # derive features from the predicted timeseries predicted_std = predicted.std(axis=0) predicted_time = np.sqrt((predicted.T**2).mean(axis=0)) VE_spatial = 1-(res.var(axis=0)/Y.var(axis=0)) VE_temporal = 1-(res.var(axis=1)/Y.var(axis=1)) if len(cr_opts.conf_list) > 0: # if confound regression is applied timeseries = res # save the temporal STD map prior to standardization and smoothing temporal_std = timeseries.std(axis=0) ''' #8 - Standardize timeseries ''' if cr_opts.standardize: timeseries = (timeseries-timeseries.mean(axis=0))/temporal_std # save output files VE_spatial_map = recover_3D(brain_mask_file, VE_spatial) STD_spatial_map = recover_3D(brain_mask_file, temporal_std) CR_STD_spatial_map = recover_3D(brain_mask_file, predicted_std) timeseries_3d = recover_4D(brain_mask_file, timeseries, bold_file) cleaned_path = cr_out+'/'+filename_split[0]+'_cleaned.nii.gz' sitk.WriteImage(timeseries_3d, cleaned_path) VE_file_path = cr_out+'/'+filename_split[0]+'_VE_map.nii.gz' sitk.WriteImage(VE_spatial_map, VE_file_path) STD_file_path = cr_out+'/'+filename_split[0]+'_STD_map.nii.gz' sitk.WriteImage(STD_spatial_map, STD_file_path) CR_STD_file_path = cr_out+'/'+filename_split[0]+'_CR_STD_map.nii.gz' sitk.WriteImage(CR_STD_spatial_map, CR_STD_file_path) frame_mask_file = cr_out+'/'+filename_split[0]+'_frame_censoring_mask.csv' pd.DataFrame(frame_mask).to_csv(frame_mask_file, index=False, header=['False = Masked Frames']) if cr_opts.smoothing_filter is not None: ''' #9 - Apply Gaussian spatial smoothing. ''' timeseries_3d = nilearn.image.smooth_img(nb.load(cleaned_path), cr_opts.smoothing_filter) timeseries_3d.to_filename(cleaned_path) # apply the frame mask to FD trace/DVARS DVARS = DVARS[frame_mask] FD_trace = FD_trace[frame_mask] # calculate temporal degrees of freedom left after confound correction num_timepoints = frame_mask.sum() if cr_opts.run_aroma: aroma_rm = (pd.read_csv(f'{aroma_out}/classification_overview.txt', sep='\t')['Motion/noise']).sum() else: aroma_rm = 0 num_regressors = confounds_array.shape[1] tDOF = num_timepoints - (aroma_rm+num_regressors) data_dict = {'FD_trace':FD_trace, 'DVARS':DVARS, 'time_range':time_range, 'frame_mask':frame_mask, 'confounds_array':confounds_array, 'VE_temporal':VE_temporal, 'confounds_csv':confounds_file, 'predicted_time':predicted_time, 'tDOF':tDOF} setattr(self, 'cleaned_path', cleaned_path) setattr(self, 'VE_file_path', VE_file_path) setattr(self, 'STD_file_path', STD_file_path) setattr(self, 'CR_STD_file_path', CR_STD_file_path) setattr(self, 'frame_mask_file', frame_mask_file) setattr(self, 'data_dict', data_dict) return runtime
def denoise(img_file, tsv_file, out_path, col_names=False, hp_filter=False, lp_filter=False, out_figure_path=False): nii_ext = '.nii.gz' FD_thr = [.5] sc_range = np.arange(-1, 3) constant = 'constant' # get file info base_file = os.path.basename(img_file) save_img_file = pjoin(out_path, base_file[0:base_file.find('.')] + \ '_NR' + nii_ext) # read in files img = load_niimg(img_file) data = img.get_data() df = pandas.read_csv(tsv_file, '\t', na_values='n/a') Ntrs = df.as_matrix().shape[0] print('# of TRs: ' + str(Ntrs)) assert (Ntrs == data.shape[len(data.shape) - 1]) # select columns to use as nuisance regressors str_append = ' [ALL regressors in CSV]' if col_names: df = df[col_names] str_append = ' [SELECTED regressors in CSV]' # fill in missing nuisance values with mean for that variable for col in df.columns: if sum(df[col].isnull()) > 0: print('Filling in ' + str(sum(df[col].isnull())) + ' NaN value for ' + col) df[col] = df[col].fillna(np.mean(df[col])) print('# of Confound Regressors: ' + str(len(df.columns)) + str_append) # implement HP filter in regression TR = img.header.get_zooms()[-1] frame_times = np.arange(Ntrs) * TR if hp_filter: hp_filter = float(hp_filter) assert (hp_filter > 0) period_cutoff = 1. / hp_filter df = make_design_matrix(frame_times, period_cut=period_cutoff, add_regs=df.as_matrix(), add_reg_names=df.columns.tolist()) # fn adds intercept into dm hp_cols = [col for col in df.columns if 'drift' in col] print('# of High-pass Filter Regressors: ' + str(len(hp_cols))) else: # add in intercept column into data frame df[constant] = 1 dm = df.as_matrix() # prep data data = np.reshape(data, (-1, Ntrs)) data_mean = np.mean(data, axis=1) Nvox = len(data_mean) # setup and run regression model = regression.OLSModel(dm) results = model.fit(data.T) if not hp_filter: results_orig_resid = copy.deepcopy( results.resid) # save for rsquared computation # apply low-pass filter if lp_filter: # input to butterworth fn is time x voxels low_pass = float(lp_filter) Fs = 1. / TR if low_pass >= Fs / 2: raise ValueError( 'Low pass filter cutoff if too close to the Nyquist frequency (%s)' % (Fs / 2)) results.resid = butterworth(results.resid, sampling_rate=Fs, low_pass=low_pass, high_pass=None) # add mean back into data clean_data = results.resid.T + np.reshape( data_mean, (Nvox, 1)) # add mean back into residuals # save out new data file clean_data = np.reshape(clean_data, img.shape).astype('float32') #new_img = nb.Nifti1Image(clean_data, img.affine) #inherit header from original new_header = header = img.header.copy() new_img = nb.Nifti1Image(clean_data, None, header=new_header) new_img.to_filename(save_img_file) ######### generate Rsquared map for confounds only if hp_filter: # first remove low-frequency information from data hp_cols.append(constant) model_first = regression.OLSModel(df[hp_cols].as_matrix()) results_first = model_first.fit(data.T) results_first_resid = copy.deepcopy(results_first.resid) del results_first, model_first # compute sst - borrowed from matlab sst = np.square( np.linalg.norm(results_first_resid - np.mean(results_first_resid, axis=0), axis=0)) # now regress out 'true' confounds to estimate their Rsquared nr_cols = [col for col in df.columns if 'drift' not in col] model_second = regression.OLSModel(df[nr_cols].as_matrix()) results_second = model_second.fit(results_first_resid) # compute sse - borrowed from matlab sse = np.square(np.linalg.norm(results_second.resid, axis=0)) del results_second, model_second, results_first_resid elif not hp_filter: # compute sst - borrowed from matlab sst = np.square( np.linalg.norm(data.T - np.mean(data.T, axis=0), axis=0)) # compute sse - borrowed from matlab sse = np.square(np.linalg.norm(results_orig_resid, axis=0)) del results_orig_resid # compute rsquared of nuisance regressors zero_idx = scipy.logical_and(sst == 0, sse == 0) sse[zero_idx] = 1 sst[zero_idx] = 1 # would be NaNs - become rsquared = 0 rsquare = 1 - np.true_divide(sse, sst) rsquare[np.isnan(rsquare)] = 0 ######### Visualizing DM & outputs fontsize = 12 fontsize_title = 14 if not out_figure_path: out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures' if not os.path.isdir(out_figure_path): os.mkdir(out_figure_path) img_name = os.path.basename(img_file) png_append = '_' + img_name[0:img_name.find('.')] + '.png' # DM corr matrix cm = df[df.columns[0:-1]].corr() mask = np.zeros_like(cm, dtype=np.bool) mask[np.triu_indices_from(mask)] = True sz = 8 if cm.shape[0] > sz: sz = sz + ((cm.shape[0] - sz) * .3) fig, ax = plt.subplots(figsize=(sz, sz)) cmap = sns.diverging_palette(220, 10, as_cmap=True) sns.heatmap(cm, mask=mask, cmap=cmap, center=0, vmax=cm[cm < 1].max().max(), vmin=cm[cm < 1].min().min(), square=True, linewidths=.5, cbar_kws={"shrink": .6}) ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize) ax.set_yticklabels(cm.columns.tolist(), rotation=-30, va='bottom', fontsize=fontsize) ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title) plt.tight_layout() fig.savefig(pjoin(out_figure_path, 'Corr_matrix_regressors' + png_append)) plt.close(fig) del fig, ax # DM of Nuisance Regressors (all) tr_label = 'TR (Volume #)' fig, ax = plt.subplots(figsize=(4, sz)) reporting.plot_design_matrix(df, ax=ax) ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title) ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize) ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize) ax.set_ylabel(tr_label, fontsize=fontsize) plt.tight_layout() fig.savefig(pjoin(out_figure_path, 'Design_matrix' + png_append)) plt.close(fig) del fig, ax # FD timeseries plot FD = 'FD' poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd'] idx = [df.columns.__contains__(i) for i in poss_names] FD_name = poss_names[idx == True] y = df[FD_name].as_matrix() Nremove = [] sc_idx = [] for thr_idx, thr in enumerate(FD_thr): idx = y >= thr sc_idx.append(copy.deepcopy(idx)) for iidx in np.where(idx)[0]: for buffer in sc_range: curr_idx = iidx + buffer if curr_idx >= 0 and curr_idx <= len(idx): sc_idx[thr_idx][curr_idx] = True Nremove.append(np.sum(sc_idx[thr_idx])) Nplots = len(FD_thr) sns.set(font_scale=1.5) sns.set_style('ticks') fig, axes = plt.subplots(Nplots, 1, figsize=(12, 4), squeeze=False) sns.despine() bound = .4 for curr in np.arange(0, Nplots): axes[curr, 0].plot(y) axes[curr, 0].plot((-bound, Ntrs + bound), FD_thr[curr] * np.ones((1, 2))[0], '--', color='black') axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20) if Nremove[curr] > 0: info = scipy.ndimage.measurements.label(sc_idx[curr]) for cluster in np.arange(1, info[1] + 1): temp = np.where(info[0] == cluster)[0] axes[curr, 0].axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red') axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')') axes[curr, 0].set_title(FD + ': ' + str(100 * Nremove[curr] / Ntrs)[0:4] + '% of scan (' + str(Nremove[curr]) + ' volumes) would be scrubbed (FD thr.= ' + str(FD_thr[curr]) + ')') plt.text(Ntrs + 1, FD_thr[curr] - .01, FD + ' = ' + str(FD_thr[curr]), fontsize=fontsize) axes[curr, 0].set_xlim((-bound, Ntrs + 8)) plt.tight_layout() axes[curr, 0].set_xlabel(tr_label) fig.savefig(pjoin(out_figure_path, FD + '_timeseries' + png_append)) plt.close(fig) del fig, axes # Display T-stat maps for nuisance regressors # create mean img img_size = (img.shape[0], img.shape[1], img.shape[2]) mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine) mx = [] for idx, col in enumerate(df.columns): if not 'drift' in col and not constant in col: con_vector = np.zeros((1, df.shape[1])) con_vector[0, idx] = 1 con = results.Tcontrast(con_vector) mx.append(np.max(np.absolute([con.t.min(), con.t.max()]))) mx = .8 * np.max(mx) t_png = 'Tstat_' for idx, col in enumerate(df.columns): if not 'drift' in col and not constant in col: con_vector = np.zeros((1, df.shape[1])) con_vector[0, idx] = 1 con = results.Tcontrast(con_vector) print(con_vector) m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine) title_str = col + ' Tstat map ' print(title_str) fig = plotting.plot_stat_map(m_img, mean_img, threshold=3, colorbar=True, display_mode='z', vmax=mx, title=title_str, cut_coords=7) fig.savefig(pjoin(out_figure_path, t_png + col + png_append)) plt.close() del fig # Display R-sq map for nuisance regressors m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine) title_str = 'Nuisance Rsq map ' print(title_str) mx = .95 * rsquare.max() fig = plotting.plot_stat_map(m_img, mean_img, threshold=.2, colorbar=True, display_mode='z', vmax=mx, title=title_str, cut_coords=7) fig.savefig(pjoin(out_figure_path, 'Rsquared' + png_append)) plt.close() del fig
def test_butterworth(): rand_gen = np.random.RandomState(0) n_features = 20000 n_samples = 100 sampling = 100 low_pass = 30 high_pass = 10 # Compare output for different options. # single timeseries data = rand_gen.randn(n_samples) data_original = data.copy() ''' May be only on py3.5: Bug in scipy 1.1.0 generates an unavoidable FutureWarning. (More info: https://github.com/scipy/scipy/issues/9086) The number of warnings generated is overwhelming TravisCI's log limit, causing it to fail tests. This hack prevents that and will be removed in future. ''' buggy_scipy = (LooseVersion(scipy.__version__) < LooseVersion('1.2') and LooseVersion(scipy.__version__) > LooseVersion('1.0') ) if buggy_scipy: warnings.simplefilter('ignore') ''' END HACK ''' out_single = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False) np.testing.assert_almost_equal(out_single, data) np.testing.assert_(id(out_single) != id(data)) # multiple timeseries data = rand_gen.randn(n_samples, n_features) data[:, 0] = data_original # set first timeseries to previous data data_original = data.copy() out1 = nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=True) np.testing.assert_almost_equal(data, data_original) np.testing.assert_(id(out1) != id(data_original)) # check that multiple- and single-timeseries filtering do the same thing. np.testing.assert_almost_equal(out1[:, 0], out_single) nisignal.butterworth(data, sampling, low_pass=low_pass, high_pass=high_pass, copy=False) np.testing.assert_almost_equal(out1, data) # Test nyquist frequency clipping, issue #482 out1 = nisignal.butterworth(data, sampling, low_pass=50., copy=True) out2 = nisignal.butterworth(data, sampling, low_pass=80., # Greater than nyq frequency copy=True) np.testing.assert_almost_equal(out1, out2) np.testing.assert_(id(out1) != id(out2))