def test_plot_carpet(testdata_4d): # noqa:F811 """Check contents of plot_carpet figure against data in image.""" img_4d = testdata_4d['img_4d'] img_4d_long = testdata_4d['img_4d_long'] mask_img = testdata_4d['img_mask'] display = plot_carpet(img_4d, mask_img, detrend=False, title='TEST') # Next two lines retrieve the numpy array from the plot ax = display.axes[0] plotted_array = ax.images[0].get_array() assert (plotted_array.shape == (np.prod(img_4d.shape[:-1]), img_4d.shape[-1])) # Make sure that the values in the figure match the values in the image np.testing.assert_almost_equal(plotted_array.sum(), img_4d.get_fdata().sum(), decimal=3) # Save execution time and memory plt.close(display) fig, ax = plt.subplots() display = plot_carpet(img_4d_long, mask_img, t_r=None, detrend=True, title='TEST', figure=fig, axes=ax) # Next two lines retrieve the numpy array from the plot ax = display.axes[0] plotted_array = ax.images[0].get_array() # Check size n_items = (np.prod(img_4d_long.shape[:-1]) * np.ceil(img_4d_long.shape[-1] / 4)) assert plotted_array.size == n_items plt.close(display)
def test_plot_carpet_with_atlas(testdata_4d): # noqa:F811 """Test plot_carpet when using an atlas.""" img_4d = testdata_4d['img_4d'] mask_img = testdata_4d['img_atlas'] atlas_labels = testdata_4d['atlas_labels'] # Test atlas - labels # t_r is set explicitly for this test as well display = plot_carpet(img_4d, mask_img, t_r=2, detrend=False, title='TEST') # Check the output # Two axes: 1 for colorbar and 1 for imshow assert len(display.axes) == 2 # The y-axis label of the imshow should be 'voxels' since atlas labels are # unknown ax = display.axes[1] assert ax.get_ylabel() == 'voxels' # Next two lines retrieve the numpy array from the plot ax = display.axes[0] colorbar = ax.images[0].get_array() assert len(np.unique(colorbar)) == len(atlas_labels) # Save execution time and memory plt.close(display) # Test atlas + labels fig, ax = plt.subplots() display = plot_carpet( img_4d, mask_img, mask_labels=atlas_labels, detrend=True, title='TEST', figure=fig, axes=ax, ) # Check the output # Two axes: 1 for colorbar and 1 for imshow assert len(display.axes) == 2 ax = display.axes[0] # The ytick labels of the colorbar should match the atlas labels yticklabels = ax.get_yticklabels() yticklabels = [yt.get_text() for yt in yticklabels] assert set(yticklabels) == set(atlas_labels.keys()) # Next two lines retrieve the numpy array from the plot ax = display.axes[0] colorbar = ax.images[0].get_array() assert len(np.unique(colorbar)) == len(atlas_labels) plt.close(display)
def carpet_plot(optcom_ts, denoised_ts, hikts, lowkts, mask, io_generator, gscontrol=None): """Generate a set of carpet plots for the combined and denoised data. Parameters ---------- optcom_ts, denoised_ts, hikts, lowkts : (S x T) array_like Different types of data to plot. mask : (S,) array-like Binary mask used to apply to the data. io_generator : :obj:`tedana.io.OutputGenerator` The output generator for this workflow gscontrol : {None, 'mir', 'gsr'} or :obj:`list`, optional Additional denoising steps applied in the workflow. If any gscontrol methods were applied, then additional carpet plots will be generated for pertinent outputs from those steps. Default is None. """ mask_img = io.new_nii_like(io_generator.reference_img, mask.astype(int)) optcom_img = io.new_nii_like(io_generator.reference_img, optcom_ts) dn_img = io.new_nii_like(io_generator.reference_img, denoised_ts) hik_img = io.new_nii_like(io_generator.reference_img, hikts) lowk_img = io.new_nii_like(io_generator.reference_img, lowkts) # Carpet plots fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( optcom_img, mask_img, figure=fig, axes=ax, title="Optimally Combined Data", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_optcom.svg")) fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( dn_img, mask_img, figure=fig, axes=ax, title="Denoised Data", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_denoised.svg")) fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( hik_img, mask_img, figure=fig, axes=ax, title="High-Kappa Data", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_accepted.svg")) fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( lowk_img, mask_img, figure=fig, axes=ax, title="Low-Kappa Data", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_rejected.svg")) if (gscontrol is not None) and ("gsr" in gscontrol): optcom_with_gs_img = io_generator.get_name("has gs combined img") fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( optcom_with_gs_img, mask_img, figure=fig, axes=ax, title="Optimally Combined Data (Pre-GSR)", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_optcom_nogsr.svg")) if (gscontrol is not None) and ("mir" in gscontrol): mir_denoised_img = io_generator.get_name("mir denoised img") fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( mir_denoised_img, mask_img, figure=fig, axes=ax, title="Denoised Data (Post-MIR)", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_denoised_mir.svg")) mir_denoised_img = io_generator.get_name( "ICA accepted mir denoised img") fig, ax = plt.subplots(figsize=(14, 7)) plotting.plot_carpet( mir_denoised_img, mask_img, figure=fig, axes=ax, title="High-Kappa Data (Post-MIR)", ) fig.tight_layout() fig.savefig( os.path.join(io_generator.out_dir, "figures", "carpet_accepted_mir.svg"))
def denoise(img_file, tsv_file, out_path, col_names=False, hp_filter=False, lp_filter=False, out_figure_path=False): nii_ext = '.nii.gz' FD_thr = [.5] sc_range = np.arange(-1, 3) constant = 'constant' # read in files img = load_niimg(img_file) # get file info img_name = os.path.basename(img.get_filename()) file_base = img_name[0:img_name.find('.')] save_img_file = pjoin(out_path, file_base + \ '_NR' + nii_ext) data = img.get_data() df_orig = pandas.read_csv(tsv_file, '\t', na_values='n/a') df = copy.deepcopy(df_orig) Ntrs = df.as_matrix().shape[0] print('# of TRs: ' + str(Ntrs)) assert (Ntrs == data.shape[len(data.shape) - 1]) # select columns to use as nuisance regressors if col_names: df = df[col_names] str_append = ' [SELECTED regressors in CSV]' else: col_names = df.columns.tolist() str_append = ' [ALL regressors in CSV]' # fill in missing nuisance values with mean for that variable for col in df.columns: if sum(df[col].isnull()) > 0: print('Filling in ' + str(sum(df[col].isnull())) + ' NaN value for ' + col) df[col] = df[col].fillna(np.mean(df[col])) print('# of Confound Regressors: ' + str(len(df.columns)) + str_append) # implement HP filter in regression TR = img.header.get_zooms()[-1] frame_times = np.arange(Ntrs) * TR if hp_filter: hp_filter = float(hp_filter) assert (hp_filter > 0) period_cutoff = 1. / hp_filter df = make_first_level_design_matrix(frame_times, period_cut=period_cutoff, add_regs=df.as_matrix(), add_reg_names=df.columns.tolist()) # fn adds intercept into dm hp_cols = [col for col in df.columns if 'drift' in col] print('# of High-pass Filter Regressors: ' + str(len(hp_cols))) else: # add in intercept column into data frame df[constant] = 1 print('No High-pass Filter Applied') dm = df.as_matrix() # prep data data = np.reshape(data, (-1, Ntrs)) data_mean = np.mean(data, axis=1) Nvox = len(data_mean) # setup and run regression model = regression.OLSModel(dm) results = model.fit(data.T) if not hp_filter: results_orig_resid = copy.deepcopy(results.resid) # save for rsquared computation # apply low-pass filter if lp_filter: # input to butterworth fn is time x voxels low_pass = float(lp_filter) Fs = 1. / TR if low_pass >= Fs / 2: raise ValueError('Low pass filter cutoff if too close to the Nyquist frequency (%s)' % (Fs / 2)) temp_img_file = pjoin(out_path, file_base + \ '_temp' + nii_ext) temp_img = nb.Nifti1Image(np.reshape(results.resid.T + np.reshape(data_mean, (Nvox, 1)), img.shape).astype('float32'), img.affine, header=img.header) temp_img.to_filename(temp_img_file) results.resid = butterworth(results.resid, sampling_rate=Fs, low_pass=low_pass, high_pass=None) print('Low-pass Filter Applied: < ' + str(low_pass) + ' Hz') # add mean back into data clean_data = results.resid.T + np.reshape(data_mean, (Nvox, 1)) # add mean back into residuals # save out new data file print('Saving output file...') clean_data = np.reshape(clean_data, img.shape).astype('float32') new_img = nb.Nifti1Image(clean_data, img.affine, header=img.header) new_img.to_filename(save_img_file) ######### generate Rsquared map for confounds only if hp_filter: # first remove low-frequency information from data hp_cols.append(constant) model_first = regression.OLSModel(df[hp_cols].as_matrix()) results_first = model_first.fit(data.T) results_first_resid = copy.deepcopy(results_first.resid) del results_first, model_first # compute sst - borrowed from matlab sst = np.square(np.linalg.norm(results_first_resid - np.mean(results_first_resid, axis=0), axis=0)) # now regress out 'true' confounds to estimate their Rsquared nr_cols = [col for col in df.columns if 'drift' not in col] model_second = regression.OLSModel(df[nr_cols].as_matrix()) results_second = model_second.fit(results_first_resid) # compute sse - borrowed from matlab sse = np.square(np.linalg.norm(results_second.resid, axis=0)) del results_second, model_second, results_first_resid elif not hp_filter: # compute sst - borrowed from matlab sst = np.square(np.linalg.norm(data.T - np.mean(data.T, axis=0), axis=0)) # compute sse - borrowed from matlab sse = np.square(np.linalg.norm(results_orig_resid, axis=0)) del results_orig_resid # compute rsquared of nuisance regressors zero_idx = scipy.logical_and(sst == 0, sse == 0) sse[zero_idx] = 1 sst[zero_idx] = 1 # would be NaNs - become rsquared = 0 rsquare = 1 - np.true_divide(sse, sst) rsquare[np.isnan(rsquare)] = 0 ######### Visualizing DM & outputs fontsize = 12 fontsize_title = 14 def_img_size = 8 if not out_figure_path: out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures' if not os.path.isdir(out_figure_path): os.mkdir(out_figure_path) png_append = '_' + img_name[0:img_name.find('.')] + '.png' print('Output directory: ' + out_figure_path) # DM corr matrix cm = df[df.columns[0:-1]].corr() curr_sz = copy.deepcopy(def_img_size) if cm.shape[0] > def_img_size: curr_sz = curr_sz + ((cm.shape[0] - curr_sz) * .3) mtx_scale = curr_sz * 100 mask = np.zeros_like(cm, dtype=np.bool) mask[np.triu_indices_from(mask)] = True fig, ax = plt.subplots(figsize=(curr_sz, curr_sz)) cmap = sns.diverging_palette(220, 10, as_cmap=True) sns.heatmap(cm, mask=mask, cmap=cmap, center=0, vmax=cm[cm < 1].max().max(), vmin=cm[cm < 1].min().min(), square=True, linewidths=.5, cbar_kws={"shrink": .6}) ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize) ax.set_yticklabels(cm.columns.tolist(), rotation=-30, va='bottom', fontsize=fontsize) ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title) plt.tight_layout() file_corr_matrix = 'Corr_matrix_regressors' + png_append fig.savefig(pjoin(out_figure_path, file_corr_matrix)) plt.close(fig) del fig, ax # DM of Nuisance Regressors (all) tr_label = 'TR (Volume #)' fig, ax = plt.subplots(figsize=(curr_sz - 4.1, def_img_size)) x_scale_html = ((curr_sz - 4.1) / def_img_size) * 890 reporting.plot_design_matrix(df, ax=ax) ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title) ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize) ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize) ax.set_ylabel(tr_label, fontsize=fontsize) plt.tight_layout() file_design_matrix = 'Design_matrix' + png_append fig.savefig(pjoin(out_figure_path, file_design_matrix)) plt.close(fig) del fig, ax # FD timeseries plot FD = 'FD' poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd'] fd_idx = [df_orig.columns.__contains__(i) for i in poss_names] if np.sum(fd_idx) > 0: FD_name = poss_names[fd_idx == True] if sum(df_orig[FD_name].isnull()) > 0: df_orig[FD_name] = df_orig[FD_name].fillna(np.mean(df_orig[FD_name])) y = df_orig[FD_name].as_matrix() Nremove = [] sc_idx = [] for thr_idx, thr in enumerate(FD_thr): idx = y >= thr sc_idx.append(copy.deepcopy(idx)) for iidx in np.where(idx)[0]: for buffer in sc_range: curr_idx = iidx + buffer if curr_idx >= 0 and curr_idx <= len(idx): sc_idx[thr_idx][curr_idx] = True Nremove.append(np.sum(sc_idx[thr_idx])) Nplots = len(FD_thr) sns.set(font_scale=1.5) sns.set_style('ticks') fig, axes = plt.subplots(Nplots, 1, figsize=(def_img_size * 1.5, def_img_size / 2), squeeze=False) sns.despine() bound = .4 fd_mean = np.mean(y) for curr in np.arange(0, Nplots): axes[curr, 0].plot(y) axes[curr, 0].plot((-bound, Ntrs + bound), FD_thr[curr] * np.ones((1, 2))[0], '--', color='black') axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20) if Nremove[curr] > 0: info = scipy.ndimage.measurements.label(sc_idx[curr]) for cluster in np.arange(1, info[1] + 1): temp = np.where(info[0] == cluster)[0] axes[curr, 0].axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red') axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')') axes[curr, 0].set_title(FD + ': ' + str(100 * Nremove[curr] / Ntrs)[0:4] + '% of scan (' + str(Nremove[curr]) + ' volumes) would be scrubbed (FD thr.= ' + str(FD_thr[curr]) + ')') plt.text(Ntrs + 1, FD_thr[curr] - .01, FD + ' = ' + str(FD_thr[curr]), fontsize=fontsize) plt.text(Ntrs, fd_mean - .01, 'avg = ' + str(fd_mean), fontsize=fontsize) axes[curr, 0].set_xlim((-bound, Ntrs + 8)) plt.tight_layout() axes[curr, 0].set_xlabel(tr_label) file_fd_plot = FD + '_timeseries' + png_append fig.savefig(pjoin(out_figure_path, file_fd_plot)) plt.close(fig) del fig, axes print(FD + ' timeseries plot saved') else: print(FD + ' not found: ' + FD + ' timeseries not plotted') file_fd_plot = None # Carpet and DVARS plots - before & after nuisance regression # need to create mask file to input to DVARS function mask_file = pjoin(out_figure_path, 'mask_temp.nii.gz') nifti_masker = NiftiMasker(mask_strategy='epi', standardize=False) nifti_masker.fit(img) nifti_masker.mask_img_.to_filename(mask_file) # create 2 or 3 carpet plots, depending on if LP filter is also applied Ncarpet = 2 total_sz = int(16) carpet_scale = 840 y_labels = ['Input (voxels)', 'Output \'cleaned\''] imgs = [img, new_img] img_files = [img_file, save_img_file] color = ['red', 'salmon'] labels = ['input', 'cleaned'] if lp_filter: Ncarpet = 3 total_sz = int(20) carpet_scale = carpet_scale * (9/8) y_labels = ['Input', 'Clean Pre-LP', 'Clean LP'] imgs.insert(1, temp_img) img_files.insert(1, temp_img_file) color.insert(1, 'firebrick') labels.insert(1, 'clean pre-LP') labels[-1] = 'clean LP' dvars = [] print('Computing dvars...') for in_file in img_files: temp = nac.compute_dvars(in_file=in_file, in_mask=mask_file)[1] dvars.append(np.hstack((temp.mean(), temp))) del temp small_sz = 2 fig = plt.figure(figsize=(def_img_size * 1.5, def_img_size + ((Ncarpet - 2) * 1))) row_used = 0 if np.sum(fd_idx) > 0: # if FD data is available row_used = row_used + small_sz ax0 = plt.subplot2grid((total_sz, 1), (0, 0), rowspan=small_sz) ax0.plot(y) ax0.scatter(np.arange(0, Ntrs), y, s=10) curr = 0 if Nremove[curr] > 0: info = scipy.ndimage.measurements.label(sc_idx[curr]) for cluster in np.arange(1, info[1] + 1): temp = np.where(info[0] == cluster)[0] ax0.axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red') ax0.set_ylabel(FD) for side in ["top", "right", "bottom"]: ax0.spines[side].set_color('none') ax0.spines[side].set_visible(False) ax0.set_xticks([]) ax0.set_xlim((-.5, Ntrs - .5)) ax0.spines["left"].set_position(('outward', 10)) ax_d = plt.subplot2grid((total_sz, 1), (row_used, 0), rowspan=small_sz) for iplot in np.arange(len(dvars)): ax_d.plot(dvars[iplot], color=color[iplot], label=labels[iplot]) ax_d.set_ylabel('DVARS') for side in ["top", "right", "bottom"]: ax_d.spines[side].set_color('none') ax_d.spines[side].set_visible(False) ax_d.set_xticks([]) ax_d.set_xlim((-.5, Ntrs - .5)) ax_d.spines["left"].set_position(('outward', 10)) ax_d.legend(fontsize=fontsize - 2) row_used = row_used + small_sz st = 0 carpet_each = int((total_sz - row_used) / Ncarpet) for idx, img_curr in enumerate(imgs): ax_curr = plt.subplot2grid((total_sz, 1), (row_used + st, 0), rowspan=carpet_each) fig = plotting.plot_carpet(img_curr, figure=fig, axes=ax_curr) ax_curr.set_ylabel(y_labels[idx]) for side in ["bottom", "left"]: ax_curr.spines[side].set_position(('outward', 10)) if idx < len(imgs)-1: ax_curr.spines["bottom"].set_visible(False) ax_curr.set_xticklabels('') ax_curr.set_xlabel('') st = st + carpet_each file_carpet_plot = 'Carpet_plots' + png_append fig.savefig(pjoin(out_figure_path, file_carpet_plot)) plt.close() del fig, ax0, ax_curr, ax_d, dvars os.remove(mask_file) print('Carpet/DVARS plots saved') if lp_filter: os.remove(temp_img_file) del temp_img # Display T-stat maps for nuisance regressors # create mean img img_size = (img.shape[0], img.shape[1], img.shape[2]) mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine) mx = [] for idx, col in enumerate(df.columns): if not 'drift' in col and not constant in col: con_vector = np.zeros((1, df.shape[1])) con_vector[0, idx] = 1 con = results.Tcontrast(con_vector) mx.append(np.max(np.absolute([con.t.min(), con.t.max()]))) mx = .8 * np.max(mx) t_png = 'Tstat_' file_tstat = [] for idx, col in enumerate(df.columns): if not 'drift' in col and not constant in col: con_vector = np.zeros((1, df.shape[1])) con_vector[0, idx] = 1 con = results.Tcontrast(con_vector) m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine) title_str = col + ' Tstat' fig = plotting.plot_stat_map(m_img, mean_img, threshold=3, colorbar=True, display_mode='z', vmax=mx, title=title_str, cut_coords=7) file_temp = t_png + col + png_append fig.savefig(pjoin(out_figure_path, file_temp)) file_tstat.append({'name': col, 'file': file_temp}) plt.close() del fig, file_temp print(title_str + ' map saved') # Display R-sq map for nuisance regressors m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine) title_str = 'Nuisance Rsq' mx = .95 * rsquare.max() fig = plotting.plot_stat_map(m_img, mean_img, threshold=.2, colorbar=True, display_mode='z', vmax=mx, title=title_str, cut_coords=7) file_rsq_map = 'Rsquared' + png_append fig.savefig(pjoin(out_figure_path, file_rsq_map)) plt.close() del fig print(title_str + ' map saved') ######### html report templateLoader = jinja2.FileSystemLoader(searchpath="/") templateEnv = jinja2.Environment(loader=templateLoader) templateVars = {"img_file": img_file, "save_img_file": save_img_file, "Ntrs": Ntrs, "tsv_file": tsv_file, "col_names": col_names, "hp_filter": hp_filter, "lp_filter": lp_filter, "file_design_matrix": file_design_matrix, "file_corr_matrix": file_corr_matrix, "file_fd_plot": file_fd_plot, "file_rsq_map": file_rsq_map, "file_tstat": file_tstat, "x_scale": x_scale_html, "mtx_scale": mtx_scale, "file_carpet_plot": file_carpet_plot, "carpet_scale": carpet_scale } TEMPLATE_FILE = pjoin(os.getcwd(), "report_template.html") template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(templateVars) html_file = pjoin(out_figure_path, img_name[0:img_name.find('.')] + '.html') with open(html_file, "w") as f: f.write(outputText) print('') print('HTML report: ' + html_file) return new_img
############################################################################### # Deriving a mask # --------------- from nilearn import masking # Build an EPI-based mask because we have no anatomical data mask_img = masking.compute_epi_mask(adhd_dataset.func[0]) ############################################################################### # Visualizing global patterns over time # ------------------------------------- import matplotlib.pyplot as plt from nilearn.plotting import plot_carpet display = plot_carpet(adhd_dataset.func[0], mask_img) display.show() ############################################################################### # Deriving a label-based mask # --------------------------- # Create a gray matter/white matter/cerebrospinal fluid mask from # ICBM152 tissue probability maps. import nibabel as nib import numpy as np from nilearn import image atlas = datasets.fetch_icbm152_2009() atlas_img = image.concat_imgs((atlas["gm"], atlas["wm"], atlas["csf"])) map_labels = {"Gray Matter": 1, "White Matter": 2, "Cerebrospinal Fluid": 3}