def plot_d_m(dm, save=False, filename='dm.png', dir_name='.', verbose=False):
"""Plot designa matrix."""
plot_design_matrix(dm)
if save:
filename = os.path.join(dir_name, filename)
if verbose:
print("Saving plot under '{0}'".format(filename))
plt.savefig(filename)
else:
plt.show()
def compute_group_z_map(second_level_input, n_sub, output_pathway):
# Model the effect of conditions (sample 1 vs sample 2).
condition_effect = np.hstack(([1] * n_sub, [- 1] * n_sub))
# Model the subject effect:
# each subject is observed in sample 1 and sample 2.
subject_effect = np.vstack((np.eye(n_sub), np.eye(n_sub)))
subjects = ['S%02d' % i for i in range(1, n_sub + 1)]
# We then assemble those in a design matrix and...
design_matrix = pd.DataFrame(
np.hstack((condition_effect[:, np.newaxis], subject_effect)),
columns=['Story vs. Math'] + subjects)
# ... plot the design_matrix.
plot_design_matrix(design_matrix, output_file=
os.path.join(output_pathway,
'design_matrix_story_math.png'))
# Specify the analysis model and fit it
second_level_model = SecondLevelModel().fit(second_level_input,
design_matrix=design_matrix)
# Estimate the contrast
z_map = second_level_model.compute_contrast('Story vs. Math',
output_type='z_score')
# Report of the GLM
report = make_glm_report(second_level_model,
contrasts='Story vs. Math',
title='Group-Level HCP900 Story vs.Math Report',
cluster_threshold=5,
height_control='fdr',
min_distance=8.,
plot_type='glass',
)
report.save_as_html(os.path.join(output_pathway, 'report.html'))
# Save contrast nifti-file
z_map.to_filename(os.path.join(output_pathway,
'group_hcplang900_story_math.nii.gz'))
# Plot contrast
threshold = 3.1 # correponds to p < .001, uncorrected
display = plotting.plot_glass_brain(z_map, threshold=threshold,
colorbar=True,
plot_abs=False,
title='Story vs. Math (unc p<0.001)',
output_file=os.path.join(
output_pathway,
'group_hcplang900_story_math'))
return z_map
def test_show_design_matrix():
# test that the show code indeed (formally) runs
frame_times = np.linspace(0, 127 * 1., 128)
dmtx = make_first_level_design_matrix(
frame_times, drift_model='polynomial', drift_order=3)
ax = plot_design_matrix(dmtx)
assert (ax is not None)
with InTemporaryDirectory():
ax = plot_design_matrix(dmtx, output_file='dmtx.png')
assert os.path.exists('dmtx.png')
assert (ax is None)
plot_design_matrix(dmtx, output_file='dmtx.pdf')
assert os.path.exists('dmtx.pdf')
def test_show_design_matrix():
# test that the show code indeed (formally) runs
frame_times = np.linspace(0, 127 * 1., 128)
DM = make_design_matrix(
frame_times, drift_model='polynomial', drift_order=3)
ax = plot_design_matrix(DM)
assert (ax is not None)
def _dmtx_to_svg_url(design_matrices):
""" Accepts a FirstLevelModel or SecondLevelModel object
with fitted design matrices & generates SVG Image URL,
which can be inserted into an HTML template.
Parameters
----------
design_matrices: List[pd.Dataframe]
Design matrices computed in the model.
Returns
-------
svg_url_design_matrices: String
SVG Image URL for the plotted design matrices,
"""
html_design_matrices = []
dmtx_template_path = os.path.join(HTML_TEMPLATE_ROOT_PATH,
'design_matrix_template.html')
with open(dmtx_template_path) as html_template_obj:
dmtx_template_text = html_template_obj.read()
for dmtx_count, design_matrix in enumerate(design_matrices, start=1):
dmtx_text_ = string.Template(dmtx_template_text)
dmtx_plot = plot_design_matrix(design_matrix)
dmtx_title = 'Session {}'.format(dmtx_count)
plt.title(dmtx_title, y=0.987)
dmtx_plot = _resize_plot_inches(dmtx_plot, height_change=.3)
url_design_matrix_svg = plot_to_svg(dmtx_plot)
# prevents sphinx-gallery & jupyter from scraping & inserting plots
plt.close()
dmtx_text_ = dmtx_text_.safe_substitute({
'design_matrix': url_design_matrix_svg,
'dmtx_title': dmtx_title,
})
html_design_matrices.append(dmtx_text_)
svg_url_design_matrices = ''.join(html_design_matrices)
return svg_url_design_matrices
#############################################################################
# Analyse data
# ------------
#
# First create an adequate design matrix with three columns: 'age',
# 'sex', 'intercept'.
import pandas as pd
import numpy as np
intercept = np.ones(n_subjects)
design_matrix = pd.DataFrame(np.vstack((age, sex, intercept)).T,
columns=['age', 'sex', 'intercept'])
#############################################################################
# Plot the design matrix
from nistats.reporting import plot_design_matrix
ax = plot_design_matrix(design_matrix)
ax.set_title('Second level design matrix', fontsize=12)
ax.set_ylabel('maps')
##########################################################################
# Specify and fit the second-level model when loading the data, we
# smooth a little bit to improve statistical behavior
from nistats.second_level_model import SecondLevelModel
second_level_model = SecondLevelModel(smoothing_fwhm=2.0, mask=mask_img)
second_level_model.fit(gray_matter_map_filenames,
design_matrix=design_matrix)
##########################################################################
# Estimate the contrast is very simple. We can just provide the column
# name of the design matrix.
#########################################################################
# Then we sample the design matrix
X2 = make_first_level_design_matrix(frame_times, events, drift_model='polynomial',
drift_order=3, hrf_model=hrf_model)
#########################################################################
# Finally we compute a FIR model
events = pd.DataFrame({'trial_type': conditions, 'onset': onsets,
'duration': duration})
hrf_model = 'FIR'
X3 = make_first_level_design_matrix(frame_times, events, hrf_model='fir',
drift_model='polynomial', drift_order=3,
fir_delays=np.arange(1, 6))
#########################################################################
# Here the three designs side by side
from nistats.reporting import plot_design_matrix
fig, (ax1, ax2, ax3) = plt.subplots(figsize=(10, 6), nrows=1, ncols=3)
plot_design_matrix(X1, ax=ax1)
ax1.set_title('Event-related design matrix', fontsize=12)
plot_design_matrix(X2, ax=ax2)
ax2.set_title('Block design matrix', fontsize=12)
plot_design_matrix(X3, ax=ax3)
ax3.set_title('FIR design matrix', fontsize=12)
#########################################################################
# Improve the layout and show the result
plt.subplots_adjust(left=0.08, top=0.9, bottom=0.21, right=0.96, wspace=0.3)
plt.show()
############################################################################
# model the subject effect: each subject is observed in sample 1 and sample 2
subject_effect = np.vstack((np.eye(n_subjects), np.eye(n_subjects)))
subjects = ['S%02d' % i for i in range(1, n_subjects + 1)]
############################################################################
# Assemble those in a design matrix
design_matrix = pd.DataFrame(np.hstack(
(condition_effect[:, np.newaxis], subject_effect)),
columns=['vertical vs horizontal'] + subjects)
############################################################################
# plot the design_matrix
from nistats.reporting import plot_design_matrix
plot_design_matrix(design_matrix)
############################################################################
# formally specify the analysis model and fit it
from nistats.second_level_model import SecondLevelModel
second_level_model = SecondLevelModel().fit(second_level_input,
design_matrix=design_matrix)
##########################################################################
# Estimating the contrast is very simple. We can just provide the column
# name of the design matrix.
z_map = second_level_model.compute_contrast('vertical vs horizontal',
output_type='z_score')
###########################################################################
def denoise(img_file, tsv_file, out_path, col_names=False, hp_filter=False, lp_filter=False, out_figure_path=False):
nii_ext = '.nii.gz'
FD_thr = [.5]
sc_range = np.arange(-1, 3)
constant = 'constant'
# read in files
img = load_niimg(img_file)
# get file info
img_name = os.path.basename(img.get_filename())
file_base = img_name[0:img_name.find('.')]
save_img_file = pjoin(out_path, file_base + \
'_NR' + nii_ext)
data = img.get_data()
df_orig = pandas.read_csv(tsv_file, '\t', na_values='n/a')
df = copy.deepcopy(df_orig)
Ntrs = df.as_matrix().shape[0]
print('# of TRs: ' + str(Ntrs))
assert (Ntrs == data.shape[len(data.shape) - 1])
# select columns to use as nuisance regressors
if col_names:
df = df[col_names]
str_append = ' [SELECTED regressors in CSV]'
else:
col_names = df.columns.tolist()
str_append = ' [ALL regressors in CSV]'
# fill in missing nuisance values with mean for that variable
for col in df.columns:
if sum(df[col].isnull()) > 0:
print('Filling in ' + str(sum(df[col].isnull())) + ' NaN value for ' + col)
df[col] = df[col].fillna(np.mean(df[col]))
print('# of Confound Regressors: ' + str(len(df.columns)) + str_append)
# implement HP filter in regression
TR = img.header.get_zooms()[-1]
frame_times = np.arange(Ntrs) * TR
if hp_filter:
hp_filter = float(hp_filter)
assert (hp_filter > 0)
period_cutoff = 1. / hp_filter
df = make_first_level_design_matrix(frame_times, period_cut=period_cutoff, add_regs=df.as_matrix(),
add_reg_names=df.columns.tolist())
# fn adds intercept into dm
hp_cols = [col for col in df.columns if 'drift' in col]
print('# of High-pass Filter Regressors: ' + str(len(hp_cols)))
else:
# add in intercept column into data frame
df[constant] = 1
print('No High-pass Filter Applied')
dm = df.as_matrix()
# prep data
data = np.reshape(data, (-1, Ntrs))
data_mean = np.mean(data, axis=1)
Nvox = len(data_mean)
# setup and run regression
model = regression.OLSModel(dm)
results = model.fit(data.T)
if not hp_filter:
results_orig_resid = copy.deepcopy(results.resid) # save for rsquared computation
# apply low-pass filter
if lp_filter:
# input to butterworth fn is time x voxels
low_pass = float(lp_filter)
Fs = 1. / TR
if low_pass >= Fs / 2:
raise ValueError('Low pass filter cutoff if too close to the Nyquist frequency (%s)' % (Fs / 2))
temp_img_file = pjoin(out_path, file_base + \
'_temp' + nii_ext)
temp_img = nb.Nifti1Image(np.reshape(results.resid.T + np.reshape(data_mean, (Nvox, 1)), img.shape).astype('float32'),
img.affine, header=img.header)
temp_img.to_filename(temp_img_file)
results.resid = butterworth(results.resid, sampling_rate=Fs, low_pass=low_pass, high_pass=None)
print('Low-pass Filter Applied: < ' + str(low_pass) + ' Hz')
# add mean back into data
clean_data = results.resid.T + np.reshape(data_mean, (Nvox, 1)) # add mean back into residuals
# save out new data file
print('Saving output file...')
clean_data = np.reshape(clean_data, img.shape).astype('float32')
new_img = nb.Nifti1Image(clean_data, img.affine, header=img.header)
new_img.to_filename(save_img_file)
######### generate Rsquared map for confounds only
if hp_filter:
# first remove low-frequency information from data
hp_cols.append(constant)
model_first = regression.OLSModel(df[hp_cols].as_matrix())
results_first = model_first.fit(data.T)
results_first_resid = copy.deepcopy(results_first.resid)
del results_first, model_first
# compute sst - borrowed from matlab
sst = np.square(np.linalg.norm(results_first_resid -
np.mean(results_first_resid, axis=0), axis=0))
# now regress out 'true' confounds to estimate their Rsquared
nr_cols = [col for col in df.columns if 'drift' not in col]
model_second = regression.OLSModel(df[nr_cols].as_matrix())
results_second = model_second.fit(results_first_resid)
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_second.resid, axis=0))
del results_second, model_second, results_first_resid
elif not hp_filter:
# compute sst - borrowed from matlab
sst = np.square(np.linalg.norm(data.T -
np.mean(data.T, axis=0), axis=0))
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_orig_resid, axis=0))
del results_orig_resid
# compute rsquared of nuisance regressors
zero_idx = scipy.logical_and(sst == 0, sse == 0)
sse[zero_idx] = 1
sst[zero_idx] = 1 # would be NaNs - become rsquared = 0
rsquare = 1 - np.true_divide(sse, sst)
rsquare[np.isnan(rsquare)] = 0
######### Visualizing DM & outputs
fontsize = 12
fontsize_title = 14
def_img_size = 8
if not out_figure_path:
out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures'
if not os.path.isdir(out_figure_path):
os.mkdir(out_figure_path)
png_append = '_' + img_name[0:img_name.find('.')] + '.png'
print('Output directory: ' + out_figure_path)
# DM corr matrix
cm = df[df.columns[0:-1]].corr()
curr_sz = copy.deepcopy(def_img_size)
if cm.shape[0] > def_img_size:
curr_sz = curr_sz + ((cm.shape[0] - curr_sz) * .3)
mtx_scale = curr_sz * 100
mask = np.zeros_like(cm, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
fig, ax = plt.subplots(figsize=(curr_sz, curr_sz))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(cm, mask=mask, cmap=cmap, center=0, vmax=cm[cm < 1].max().max(), vmin=cm[cm < 1].min().min(),
square=True, linewidths=.5, cbar_kws={"shrink": .6})
ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize)
ax.set_yticklabels(cm.columns.tolist(), rotation=-30, va='bottom', fontsize=fontsize)
ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title)
plt.tight_layout()
file_corr_matrix = 'Corr_matrix_regressors' + png_append
fig.savefig(pjoin(out_figure_path, file_corr_matrix))
plt.close(fig)
del fig, ax
# DM of Nuisance Regressors (all)
tr_label = 'TR (Volume #)'
fig, ax = plt.subplots(figsize=(curr_sz - 4.1, def_img_size))
x_scale_html = ((curr_sz - 4.1) / def_img_size) * 890
reporting.plot_design_matrix(df, ax=ax)
ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title)
ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize)
ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
ax.set_ylabel(tr_label, fontsize=fontsize)
plt.tight_layout()
file_design_matrix = 'Design_matrix' + png_append
fig.savefig(pjoin(out_figure_path, file_design_matrix))
plt.close(fig)
del fig, ax
# FD timeseries plot
FD = 'FD'
poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd']
fd_idx = [df_orig.columns.__contains__(i) for i in poss_names]
if np.sum(fd_idx) > 0:
FD_name = poss_names[fd_idx == True]
if sum(df_orig[FD_name].isnull()) > 0:
df_orig[FD_name] = df_orig[FD_name].fillna(np.mean(df_orig[FD_name]))
y = df_orig[FD_name].as_matrix()
Nremove = []
sc_idx = []
for thr_idx, thr in enumerate(FD_thr):
idx = y >= thr
sc_idx.append(copy.deepcopy(idx))
for iidx in np.where(idx)[0]:
for buffer in sc_range:
curr_idx = iidx + buffer
if curr_idx >= 0 and curr_idx <= len(idx):
sc_idx[thr_idx][curr_idx] = True
Nremove.append(np.sum(sc_idx[thr_idx]))
Nplots = len(FD_thr)
sns.set(font_scale=1.5)
sns.set_style('ticks')
fig, axes = plt.subplots(Nplots, 1, figsize=(def_img_size * 1.5, def_img_size / 2), squeeze=False)
sns.despine()
bound = .4
fd_mean = np.mean(y)
for curr in np.arange(0, Nplots):
axes[curr, 0].plot(y)
axes[curr, 0].plot((-bound, Ntrs + bound), FD_thr[curr] * np.ones((1, 2))[0], '--', color='black')
axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20)
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
axes[curr, 0].axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red')
axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')')
axes[curr, 0].set_title(FD + ': ' + str(100 * Nremove[curr] / Ntrs)[0:4]
+ '% of scan (' + str(Nremove[curr]) + ' volumes) would be scrubbed (FD thr.= ' +
str(FD_thr[curr]) + ')')
plt.text(Ntrs + 1, FD_thr[curr] - .01, FD + ' = ' + str(FD_thr[curr]), fontsize=fontsize)
plt.text(Ntrs, fd_mean - .01, 'avg = ' + str(fd_mean), fontsize=fontsize)
axes[curr, 0].set_xlim((-bound, Ntrs + 8))
plt.tight_layout()
axes[curr, 0].set_xlabel(tr_label)
file_fd_plot = FD + '_timeseries' + png_append
fig.savefig(pjoin(out_figure_path, file_fd_plot))
plt.close(fig)
del fig, axes
print(FD + ' timeseries plot saved')
else:
print(FD + ' not found: ' + FD + ' timeseries not plotted')
file_fd_plot = None
# Carpet and DVARS plots - before & after nuisance regression
# need to create mask file to input to DVARS function
mask_file = pjoin(out_figure_path, 'mask_temp.nii.gz')
nifti_masker = NiftiMasker(mask_strategy='epi', standardize=False)
nifti_masker.fit(img)
nifti_masker.mask_img_.to_filename(mask_file)
# create 2 or 3 carpet plots, depending on if LP filter is also applied
Ncarpet = 2
total_sz = int(16)
carpet_scale = 840
y_labels = ['Input (voxels)', 'Output \'cleaned\'']
imgs = [img, new_img]
img_files = [img_file, save_img_file]
color = ['red', 'salmon']
labels = ['input', 'cleaned']
if lp_filter:
Ncarpet = 3
total_sz = int(20)
carpet_scale = carpet_scale * (9/8)
y_labels = ['Input', 'Clean Pre-LP', 'Clean LP']
imgs.insert(1, temp_img)
img_files.insert(1, temp_img_file)
color.insert(1, 'firebrick')
labels.insert(1, 'clean pre-LP')
labels[-1] = 'clean LP'
dvars = []
print('Computing dvars...')
for in_file in img_files:
temp = nac.compute_dvars(in_file=in_file, in_mask=mask_file)[1]
dvars.append(np.hstack((temp.mean(), temp)))
del temp
small_sz = 2
fig = plt.figure(figsize=(def_img_size * 1.5, def_img_size + ((Ncarpet - 2) * 1)))
row_used = 0
if np.sum(fd_idx) > 0: # if FD data is available
row_used = row_used + small_sz
ax0 = plt.subplot2grid((total_sz, 1), (0, 0), rowspan=small_sz)
ax0.plot(y)
ax0.scatter(np.arange(0, Ntrs), y, s=10)
curr = 0
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
ax0.axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red')
ax0.set_ylabel(FD)
for side in ["top", "right", "bottom"]:
ax0.spines[side].set_color('none')
ax0.spines[side].set_visible(False)
ax0.set_xticks([])
ax0.set_xlim((-.5, Ntrs - .5))
ax0.spines["left"].set_position(('outward', 10))
ax_d = plt.subplot2grid((total_sz, 1), (row_used, 0), rowspan=small_sz)
for iplot in np.arange(len(dvars)):
ax_d.plot(dvars[iplot], color=color[iplot], label=labels[iplot])
ax_d.set_ylabel('DVARS')
for side in ["top", "right", "bottom"]:
ax_d.spines[side].set_color('none')
ax_d.spines[side].set_visible(False)
ax_d.set_xticks([])
ax_d.set_xlim((-.5, Ntrs - .5))
ax_d.spines["left"].set_position(('outward', 10))
ax_d.legend(fontsize=fontsize - 2)
row_used = row_used + small_sz
st = 0
carpet_each = int((total_sz - row_used) / Ncarpet)
for idx, img_curr in enumerate(imgs):
ax_curr = plt.subplot2grid((total_sz, 1), (row_used + st, 0), rowspan=carpet_each)
fig = plotting.plot_carpet(img_curr, figure=fig, axes=ax_curr)
ax_curr.set_ylabel(y_labels[idx])
for side in ["bottom", "left"]:
ax_curr.spines[side].set_position(('outward', 10))
if idx < len(imgs)-1:
ax_curr.spines["bottom"].set_visible(False)
ax_curr.set_xticklabels('')
ax_curr.set_xlabel('')
st = st + carpet_each
file_carpet_plot = 'Carpet_plots' + png_append
fig.savefig(pjoin(out_figure_path, file_carpet_plot))
plt.close()
del fig, ax0, ax_curr, ax_d, dvars
os.remove(mask_file)
print('Carpet/DVARS plots saved')
if lp_filter:
os.remove(temp_img_file)
del temp_img
# Display T-stat maps for nuisance regressors
# create mean img
img_size = (img.shape[0], img.shape[1], img.shape[2])
mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine)
mx = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
mx.append(np.max(np.absolute([con.t.min(), con.t.max()])))
mx = .8 * np.max(mx)
t_png = 'Tstat_'
file_tstat = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine)
title_str = col + ' Tstat'
fig = plotting.plot_stat_map(m_img, mean_img, threshold=3, colorbar=True, display_mode='z', vmax=mx,
title=title_str,
cut_coords=7)
file_temp = t_png + col + png_append
fig.savefig(pjoin(out_figure_path, file_temp))
file_tstat.append({'name': col, 'file': file_temp})
plt.close()
del fig, file_temp
print(title_str + ' map saved')
# Display R-sq map for nuisance regressors
m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine)
title_str = 'Nuisance Rsq'
mx = .95 * rsquare.max()
fig = plotting.plot_stat_map(m_img, mean_img, threshold=.2, colorbar=True, display_mode='z', vmax=mx,
title=title_str,
cut_coords=7)
file_rsq_map = 'Rsquared' + png_append
fig.savefig(pjoin(out_figure_path, file_rsq_map))
plt.close()
del fig
print(title_str + ' map saved')
######### html report
templateLoader = jinja2.FileSystemLoader(searchpath="/")
templateEnv = jinja2.Environment(loader=templateLoader)
templateVars = {"img_file": img_file,
"save_img_file": save_img_file,
"Ntrs": Ntrs,
"tsv_file": tsv_file,
"col_names": col_names,
"hp_filter": hp_filter,
"lp_filter": lp_filter,
"file_design_matrix": file_design_matrix,
"file_corr_matrix": file_corr_matrix,
"file_fd_plot": file_fd_plot,
"file_rsq_map": file_rsq_map,
"file_tstat": file_tstat,
"x_scale": x_scale_html,
"mtx_scale": mtx_scale,
"file_carpet_plot": file_carpet_plot,
"carpet_scale": carpet_scale
}
TEMPLATE_FILE = pjoin(os.getcwd(), "report_template.html")
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(templateVars)
html_file = pjoin(out_figure_path, img_name[0:img_name.find('.')] + '.html')
with open(html_file, "w") as f:
f.write(outputText)
print('')
print('HTML report: ' + html_file)
return new_img
period_cut=160)
###############################################################################
# Now that we have specified the model, we can run it on the fMRI image
fmri_glm = fmri_glm.fit(fmri_img, events)
###############################################################################
# One can inspect the design matrix (rows represent time, and
# columns contain the predictors).
design_matrix = fmri_glm.design_matrices_[0]
###############################################################################
# Formally, we have taken the first design matrix, because the model is
# implictily meant to for multiple runs.
from nistats.reporting import plot_design_matrix
plot_design_matrix(design_matrix)
import matplotlib.pyplot as plt
plt.show()
###############################################################################
# Save the design matrix image to disk
# first create a directory where you want to write the images
import os
outdir = 'results'
if not os.path.exists(outdir):
os.mkdir(outdir)
from os.path import join
plot_design_matrix(design_matrix, output_file=join(outdir, 'design_matrix.png'))
period_cut=160)
###############################################################################
# Now that we have specified the model, we can run it on the fMRI image
fmri_glm = fmri_glm.fit(fmri_img, events)
###############################################################################
# One can inspect the design matrix (rows represent time, and
# columns contain the predictors).
design_matrix = fmri_glm.design_matrices_[0]
###############################################################################
# Formally, we have taken the first design matrix, because the model is
# implictily meant to for multiple runs.
from nistats.reporting import plot_design_matrix
plot_design_matrix(design_matrix)
import matplotlib.pyplot as plt
plt.show()
###############################################################################
# Save the design matrix image to disk
# first create a directory where you want to write the images
import os
outdir = 'results'
if not os.path.exists(outdir):
os.mkdir(outdir)
from os.path import join
plot_design_matrix(design_matrix,
output_file=join(outdir, 'design_matrix.png'))
#--- definng the model
fmri_glm = FirstLevelModel(
tr,
noise_model='ar1',
standardize=False,
hrf_model='spm',
drift_model='cosine',
period_cut=128,
)
fmri_glm_non_smoothed = fmri_glm.fit(fmri,
events,
confounds=confounds_clean)
design_matrix = fmri_glm_non_smoothed.design_matrices_[0]
fig, ax = plt.subplots(figsize=(15, 8))
plot_design_matrix(design_matrix, ax=ax)
fig.savefig(f'{out_dir}{sub}/figures/{sub}_design_matrix.png')
#--- defining contrasts
trial_dummies = pd.get_dummies(design_matrix.columns)
trial_contrasts = pd.DataFrame.to_dict(trial_dummies, orient='list')
trial_contrasts['overall'] = [
1 if i[0] == 'Q' else 0 for i in design_matrix.columns
]
fig, ax = plt.subplots(n_trials, 1)
fig.set_size_inches(22, 40)
for i in range(n_trials):
plt.figure()
_ = plot_contrast_matrix(trial_contrasts[f'Q{i+1:02}'],
for r in range(par_offset):
contrast_motion[r, 6 + r] = 1
contrast_list['con_F_motion'] = contrast_motion
# Estimate and plot contrasts
for cont in contrast_list:
z_map = fmri_glm.compute_contrast(contrast_list[cont],
stat_type=cont[4],
output_type='z_score')
identifier = '%s_sub-%02d_%s' % (cont, sdx, method)
out_file = '%s/zmap_%s' % (res_path, identifier)
z_map.to_filename('%s.nii.gz' % out_file)
# Plot design matrices and contrasts
plot_design_matrix(fmri_glm.design_matrices_[0],
output_file='%s/design_matrix_01_sub-%02d.svg' %
(res_path, sdx))
plot_design_matrix(fmri_glm.design_matrices_[1],
output_file='%s/design_matrix_02_sub-%02d.svg' %
(res_path, sdx))
plot_design_matrix(fmri_glm.design_matrices_[2],
output_file='%s/design_matrix_03_sub-%02d.svg' %
(res_path, sdx))
plot_design_matrix(fmri_glm.design_matrices_[3],
output_file='%s/design_matrix_04_sub-%02d.svg' %
(res_path, sdx))
# Plot contrast matrix
confounds = pd.DataFrame(high_variance_confounds(fmri_img, percentile=1))
fmri_glm = FirstLevelModel(t_r=2.5,
noise_model='ar1',
standardize=False,
hrf_model='spm',
drift_model='cosine',
period_cut=160,
smoothing_fwhm=smoothing)
fmri_glm = fmri_glm.fit(fmri_img, events, confounds=confounds)
design_matrix = fmri_glm.design_matrices_[0]
# Save the design matrix image to disk
if not os.path.exists(outdir): os.mkdir(outdir)
plot_design_matrix(design_matrix,
output_file=join(
outdir,
subject + '_block_%s_' % '_'.join(map(str, run_order)) +
'_design_matrix.png'))
plt.close()
print('Design matrix plot saved to: ' + join(
outdir, subject + '_block_%s_' % '_'.join(map(str, run_order)) +
'_design_matrix.png'))
# The first column contains the expected reponse profile of regions which are sensitive to the stimuli in the condition
if cond1 in design_matrix.columns:
plt.plot(design_matrix[cond1])
plt.xlabel('Time (sec)')
plt.title('Expected ' + cond1 + ' Response')
plt.savefig(join(outdir, cond1 + '_expected_response.png'))
###############################################################################
# Detecting voxels with significant effects
def denoise(img_file,
tsv_file,
out_path,
col_names=False,
hp_filter=False,
lp_filter=False,
out_figure_path=False):
nii_ext = '.nii.gz'
FD_thr = [.5]
sc_range = np.arange(-1, 3)
constant = 'constant'
# get file info
base_file = os.path.basename(img_file)
save_img_file = pjoin(out_path, base_file[0:base_file.find('.')] + \
'_NR' + nii_ext)
# read in files
img = load_niimg(img_file)
data = img.get_data()
df = pandas.read_csv(tsv_file, '\t', na_values='n/a')
Ntrs = df.as_matrix().shape[0]
print('# of TRs: ' + str(Ntrs))
assert (Ntrs == data.shape[len(data.shape) - 1])
# select columns to use as nuisance regressors
str_append = ' [ALL regressors in CSV]'
if col_names:
df = df[col_names]
str_append = ' [SELECTED regressors in CSV]'
# fill in missing nuisance values with mean for that variable
for col in df.columns:
if sum(df[col].isnull()) > 0:
print('Filling in ' + str(sum(df[col].isnull())) +
' NaN value for ' + col)
df[col] = df[col].fillna(np.mean(df[col]))
print('# of Confound Regressors: ' + str(len(df.columns)) + str_append)
# implement HP filter in regression
TR = img.header.get_zooms()[-1]
frame_times = np.arange(Ntrs) * TR
if hp_filter:
hp_filter = float(hp_filter)
assert (hp_filter > 0)
period_cutoff = 1. / hp_filter
df = make_design_matrix(frame_times,
period_cut=period_cutoff,
add_regs=df.as_matrix(),
add_reg_names=df.columns.tolist())
# fn adds intercept into dm
hp_cols = [col for col in df.columns if 'drift' in col]
print('# of High-pass Filter Regressors: ' + str(len(hp_cols)))
else:
# add in intercept column into data frame
df[constant] = 1
dm = df.as_matrix()
# prep data
data = np.reshape(data, (-1, Ntrs))
data_mean = np.mean(data, axis=1)
Nvox = len(data_mean)
# setup and run regression
model = regression.OLSModel(dm)
results = model.fit(data.T)
if not hp_filter:
results_orig_resid = copy.deepcopy(
results.resid) # save for rsquared computation
# apply low-pass filter
if lp_filter:
# input to butterworth fn is time x voxels
low_pass = float(lp_filter)
Fs = 1. / TR
if low_pass >= Fs / 2:
raise ValueError(
'Low pass filter cutoff if too close to the Nyquist frequency (%s)'
% (Fs / 2))
results.resid = butterworth(results.resid,
sampling_rate=Fs,
low_pass=low_pass,
high_pass=None)
# add mean back into data
clean_data = results.resid.T + np.reshape(
data_mean, (Nvox, 1)) # add mean back into residuals
# save out new data file
clean_data = np.reshape(clean_data, img.shape).astype('float32')
#new_img = nb.Nifti1Image(clean_data, img.affine)
#inherit header from original
new_header = header = img.header.copy()
new_img = nb.Nifti1Image(clean_data, None, header=new_header)
new_img.to_filename(save_img_file)
######### generate Rsquared map for confounds only
if hp_filter:
# first remove low-frequency information from data
hp_cols.append(constant)
model_first = regression.OLSModel(df[hp_cols].as_matrix())
results_first = model_first.fit(data.T)
results_first_resid = copy.deepcopy(results_first.resid)
del results_first, model_first
# compute sst - borrowed from matlab
sst = np.square(
np.linalg.norm(results_first_resid -
np.mean(results_first_resid, axis=0),
axis=0))
# now regress out 'true' confounds to estimate their Rsquared
nr_cols = [col for col in df.columns if 'drift' not in col]
model_second = regression.OLSModel(df[nr_cols].as_matrix())
results_second = model_second.fit(results_first_resid)
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_second.resid, axis=0))
del results_second, model_second, results_first_resid
elif not hp_filter:
# compute sst - borrowed from matlab
sst = np.square(
np.linalg.norm(data.T - np.mean(data.T, axis=0), axis=0))
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_orig_resid, axis=0))
del results_orig_resid
# compute rsquared of nuisance regressors
zero_idx = scipy.logical_and(sst == 0, sse == 0)
sse[zero_idx] = 1
sst[zero_idx] = 1 # would be NaNs - become rsquared = 0
rsquare = 1 - np.true_divide(sse, sst)
rsquare[np.isnan(rsquare)] = 0
######### Visualizing DM & outputs
fontsize = 12
fontsize_title = 14
if not out_figure_path:
out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures'
if not os.path.isdir(out_figure_path):
os.mkdir(out_figure_path)
img_name = os.path.basename(img_file)
png_append = '_' + img_name[0:img_name.find('.')] + '.png'
# DM corr matrix
cm = df[df.columns[0:-1]].corr()
mask = np.zeros_like(cm, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
sz = 8
if cm.shape[0] > sz:
sz = sz + ((cm.shape[0] - sz) * .3)
fig, ax = plt.subplots(figsize=(sz, sz))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(cm,
mask=mask,
cmap=cmap,
center=0,
vmax=cm[cm < 1].max().max(),
vmin=cm[cm < 1].min().min(),
square=True,
linewidths=.5,
cbar_kws={"shrink": .6})
ax.set_xticklabels(ax.get_xticklabels(),
rotation=60,
ha='right',
fontsize=fontsize)
ax.set_yticklabels(cm.columns.tolist(),
rotation=-30,
va='bottom',
fontsize=fontsize)
ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title)
plt.tight_layout()
fig.savefig(pjoin(out_figure_path, 'Corr_matrix_regressors' + png_append))
plt.close(fig)
del fig, ax
# DM of Nuisance Regressors (all)
tr_label = 'TR (Volume #)'
fig, ax = plt.subplots(figsize=(4, sz))
reporting.plot_design_matrix(df, ax=ax)
ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title)
ax.set_xticklabels(ax.get_xticklabels(),
rotation=60,
ha='right',
fontsize=fontsize)
ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
ax.set_ylabel(tr_label, fontsize=fontsize)
plt.tight_layout()
fig.savefig(pjoin(out_figure_path, 'Design_matrix' + png_append))
plt.close(fig)
del fig, ax
# FD timeseries plot
FD = 'FD'
poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd']
idx = [df.columns.__contains__(i) for i in poss_names]
FD_name = poss_names[idx == True]
y = df[FD_name].as_matrix()
Nremove = []
sc_idx = []
for thr_idx, thr in enumerate(FD_thr):
idx = y >= thr
sc_idx.append(copy.deepcopy(idx))
for iidx in np.where(idx)[0]:
for buffer in sc_range:
curr_idx = iidx + buffer
if curr_idx >= 0 and curr_idx <= len(idx):
sc_idx[thr_idx][curr_idx] = True
Nremove.append(np.sum(sc_idx[thr_idx]))
Nplots = len(FD_thr)
sns.set(font_scale=1.5)
sns.set_style('ticks')
fig, axes = plt.subplots(Nplots, 1, figsize=(12, 4), squeeze=False)
sns.despine()
bound = .4
for curr in np.arange(0, Nplots):
axes[curr, 0].plot(y)
axes[curr, 0].plot((-bound, Ntrs + bound),
FD_thr[curr] * np.ones((1, 2))[0],
'--',
color='black')
axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20)
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
axes[curr, 0].axvspan(temp.min() - bound,
temp.max() + bound,
alpha=.5,
color='red')
axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')')
axes[curr, 0].set_title(FD + ': ' +
str(100 * Nremove[curr] / Ntrs)[0:4] +
'% of scan (' + str(Nremove[curr]) +
' volumes) would be scrubbed (FD thr.= ' +
str(FD_thr[curr]) + ')')
plt.text(Ntrs + 1,
FD_thr[curr] - .01,
FD + ' = ' + str(FD_thr[curr]),
fontsize=fontsize)
axes[curr, 0].set_xlim((-bound, Ntrs + 8))
plt.tight_layout()
axes[curr, 0].set_xlabel(tr_label)
fig.savefig(pjoin(out_figure_path, FD + '_timeseries' + png_append))
plt.close(fig)
del fig, axes
# Display T-stat maps for nuisance regressors
# create mean img
img_size = (img.shape[0], img.shape[1], img.shape[2])
mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine)
mx = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
mx.append(np.max(np.absolute([con.t.min(), con.t.max()])))
mx = .8 * np.max(mx)
t_png = 'Tstat_'
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
print(con_vector)
m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine)
title_str = col + ' Tstat map '
print(title_str)
fig = plotting.plot_stat_map(m_img,
mean_img,
threshold=3,
colorbar=True,
display_mode='z',
vmax=mx,
title=title_str,
cut_coords=7)
fig.savefig(pjoin(out_figure_path, t_png + col + png_append))
plt.close()
del fig
# Display R-sq map for nuisance regressors
m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine)
title_str = 'Nuisance Rsq map '
print(title_str)
mx = .95 * rsquare.max()
fig = plotting.plot_stat_map(m_img,
mean_img,
threshold=.2,
colorbar=True,
display_mode='z',
vmax=mx,
title=title_str,
cut_coords=7)
fig.savefig(pjoin(out_figure_path, 'Rsquared' + png_append))
plt.close()
del fig