def test_butterworth():
rand_gen = np.random.RandomState(0)
n_features = 20000
n_samples = 100
sampling = 100
low_pass = 30
high_pass = 10
# Compare output for different options.
# single timeseries
data = rand_gen.randn(n_samples)
data_original = data.copy()
out_single = nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=False,
save_memory=True)
np.testing.assert_almost_equal(out_single, data)
# multiple timeseries
data = rand_gen.randn(n_samples, n_features)
data[:, 0] = data_original # set first timeseries to previous data
data_original = data.copy()
out1 = nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
# check that multiple- and single-timeseries filtering do the same thing.
np.testing.assert_almost_equal(out1[:, 0], out_single)
nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=False)
np.testing.assert_almost_equal(out1, data)
# Test nyquist frequency clipping, issue #482
out1 = nisignal.butterworth(data, sampling, low_pass=50., copy=True)
out2 = nisignal.butterworth(
data,
sampling,
low_pass=80., # Greater than nyq frequency
copy=True)
np.testing.assert_almost_equal(out1, out2)
def test_butterworth():
rand_gen = np.random.RandomState(0)
n_features = 20000
n_samples = 100
sampling = 100
low_pass = 30
high_pass = 10
# Compare output for different options.
# single timeseries
data = rand_gen.randn(n_samples)
data_original = data.copy()
out_single = nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=False, save_memory=True)
np.testing.assert_almost_equal(out_single, data)
# multiple timeseries
data = rand_gen.randn(n_samples, n_features)
data[:, 0] = data_original # set first timeseries to previous data
data_original = data.copy()
out1 = nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
# check that multiple- and single-timeseries filtering do the same thing.
np.testing.assert_almost_equal(out1[:, 0], out_single)
nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=False)
np.testing.assert_almost_equal(out1, data)
# Test nyquist frequency clipping, issue #482
out1 = nisignal.butterworth(data, sampling,
low_pass=50.,
copy=True)
out2 = nisignal.butterworth(data, sampling,
low_pass=80., # Greater than nyq frequency
copy=True)
np.testing.assert_almost_equal(out1, out2)
def burgess_process(cifti_file, nifti_file, motion_file, hp, work_dir,
out_file):
"""regress out motion from a cifti and filter"""
cii = nib.load(cifti_file)
data = cii.get_fdata()
tr = cii.header.matrix.get_index_map(0).series_step
motion = np.loadtxt(motion_file)
confounds = motion_confounds(motion, hp, nifti_file, work_dir)
data = data - confounds.dot(np.linalg.pinv(confounds, 1e-6).dot(data))
data = signal.butterworth(data, 1 / tr, high_pass=0.009, copy=True)
out_cii = nib.cifti2.cifti2.Cifti2Image(data, cii.header, cii.nifti_header)
out_cii.to_filename(out_file)
return data
def filtered(PATH_GZ, Time_R):
from nilearn.input_data import NiftiMasker
from nilearn.signal import butterworth
import os
masker = NiftiMasker()
signal = masker.fit_transform(PATH_GZ)
X_filtered = butterworth(signals=signal,
sampling_rate=1. / Time_R,
high_pass=0.01,
copy=True)
fmri_filtered = masker.inverse_transform(X_filtered)
OutFile = 'f' + PATH_GZ[PATH_GZ.rfind('/') + 1:]
fmri_filtered.to_filename(OutFile)
out_file = os.path.abspath(OutFile)
return out_file
def test_butterworth():
rng = np.random.RandomState(42)
n_features = 20000
n_samples = 100
sampling = 100
low_pass = 30
high_pass = 10
# Compare output for different options.
# single timeseries
data = rng.standard_normal(size=n_samples)
data_original = data.copy()
'''
May be only on py3.5:
Bug in scipy 1.1.0 generates an unavoidable FutureWarning.
(More info: https://github.com/scipy/scipy/issues/9086)
The number of warnings generated is overwhelming TravisCI's log limit,
causing it to fail tests.
This hack prevents that and will be removed in future.
'''
buggy_scipy = (LooseVersion(scipy.__version__) < LooseVersion('1.2')
and LooseVersion(scipy.__version__) > LooseVersion('1.0'))
if buggy_scipy:
warnings.simplefilter('ignore')
''' END HACK '''
out_single = nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=False)
np.testing.assert_almost_equal(out_single, data)
np.testing.assert_(id(out_single) != id(data))
# multiple timeseries
data = rng.standard_normal(size=(n_samples, n_features))
data[:, 0] = data_original # set first timeseries to previous data
data_original = data.copy()
out1 = nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
np.testing.assert_(id(out1) != id(data_original))
# check that multiple- and single-timeseries filtering do the same thing.
np.testing.assert_almost_equal(out1[:, 0], out_single)
nisignal.butterworth(data,
sampling,
low_pass=low_pass,
high_pass=high_pass,
copy=False)
np.testing.assert_almost_equal(out1, data)
# Test nyquist frequency clipping, issue #482
out1 = nisignal.butterworth(data, sampling, low_pass=50., copy=True)
out2 = nisignal.butterworth(
data,
sampling,
low_pass=80., # Greater than nyq frequency
copy=True)
np.testing.assert_almost_equal(out1, out2)
np.testing.assert_(id(out1) != id(out2))
def denoise(img_file, tsv_file, out_path, col_names=False, hp_filter=False, lp_filter=False, out_figure_path=False):
nii_ext = '.nii.gz'
FD_thr = [.5]
sc_range = np.arange(-1, 3)
constant = 'constant'
# read in files
img = load_niimg(img_file)
# get file info
img_name = os.path.basename(img.get_filename())
file_base = img_name[0:img_name.find('.')]
save_img_file = pjoin(out_path, file_base + \
'_NR' + nii_ext)
data = img.get_data()
df_orig = pandas.read_csv(tsv_file, '\t', na_values='n/a')
df = copy.deepcopy(df_orig)
Ntrs = df.as_matrix().shape[0]
print('# of TRs: ' + str(Ntrs))
assert (Ntrs == data.shape[len(data.shape) - 1])
# select columns to use as nuisance regressors
if col_names:
df = df[col_names]
str_append = ' [SELECTED regressors in CSV]'
else:
col_names = df.columns.tolist()
str_append = ' [ALL regressors in CSV]'
# fill in missing nuisance values with mean for that variable
for col in df.columns:
if sum(df[col].isnull()) > 0:
print('Filling in ' + str(sum(df[col].isnull())) + ' NaN value for ' + col)
df[col] = df[col].fillna(np.mean(df[col]))
print('# of Confound Regressors: ' + str(len(df.columns)) + str_append)
# implement HP filter in regression
TR = img.header.get_zooms()[-1]
frame_times = np.arange(Ntrs) * TR
if hp_filter:
hp_filter = float(hp_filter)
assert (hp_filter > 0)
period_cutoff = 1. / hp_filter
df = make_first_level_design_matrix(frame_times, period_cut=period_cutoff, add_regs=df.as_matrix(),
add_reg_names=df.columns.tolist())
# fn adds intercept into dm
hp_cols = [col for col in df.columns if 'drift' in col]
print('# of High-pass Filter Regressors: ' + str(len(hp_cols)))
else:
# add in intercept column into data frame
df[constant] = 1
print('No High-pass Filter Applied')
dm = df.as_matrix()
# prep data
data = np.reshape(data, (-1, Ntrs))
data_mean = np.mean(data, axis=1)
Nvox = len(data_mean)
# setup and run regression
model = regression.OLSModel(dm)
results = model.fit(data.T)
if not hp_filter:
results_orig_resid = copy.deepcopy(results.resid) # save for rsquared computation
# apply low-pass filter
if lp_filter:
# input to butterworth fn is time x voxels
low_pass = float(lp_filter)
Fs = 1. / TR
if low_pass >= Fs / 2:
raise ValueError('Low pass filter cutoff if too close to the Nyquist frequency (%s)' % (Fs / 2))
temp_img_file = pjoin(out_path, file_base + \
'_temp' + nii_ext)
temp_img = nb.Nifti1Image(np.reshape(results.resid.T + np.reshape(data_mean, (Nvox, 1)), img.shape).astype('float32'),
img.affine, header=img.header)
temp_img.to_filename(temp_img_file)
results.resid = butterworth(results.resid, sampling_rate=Fs, low_pass=low_pass, high_pass=None)
print('Low-pass Filter Applied: < ' + str(low_pass) + ' Hz')
# add mean back into data
clean_data = results.resid.T + np.reshape(data_mean, (Nvox, 1)) # add mean back into residuals
# save out new data file
print('Saving output file...')
clean_data = np.reshape(clean_data, img.shape).astype('float32')
new_img = nb.Nifti1Image(clean_data, img.affine, header=img.header)
new_img.to_filename(save_img_file)
######### generate Rsquared map for confounds only
if hp_filter:
# first remove low-frequency information from data
hp_cols.append(constant)
model_first = regression.OLSModel(df[hp_cols].as_matrix())
results_first = model_first.fit(data.T)
results_first_resid = copy.deepcopy(results_first.resid)
del results_first, model_first
# compute sst - borrowed from matlab
sst = np.square(np.linalg.norm(results_first_resid -
np.mean(results_first_resid, axis=0), axis=0))
# now regress out 'true' confounds to estimate their Rsquared
nr_cols = [col for col in df.columns if 'drift' not in col]
model_second = regression.OLSModel(df[nr_cols].as_matrix())
results_second = model_second.fit(results_first_resid)
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_second.resid, axis=0))
del results_second, model_second, results_first_resid
elif not hp_filter:
# compute sst - borrowed from matlab
sst = np.square(np.linalg.norm(data.T -
np.mean(data.T, axis=0), axis=0))
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_orig_resid, axis=0))
del results_orig_resid
# compute rsquared of nuisance regressors
zero_idx = scipy.logical_and(sst == 0, sse == 0)
sse[zero_idx] = 1
sst[zero_idx] = 1 # would be NaNs - become rsquared = 0
rsquare = 1 - np.true_divide(sse, sst)
rsquare[np.isnan(rsquare)] = 0
######### Visualizing DM & outputs
fontsize = 12
fontsize_title = 14
def_img_size = 8
if not out_figure_path:
out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures'
if not os.path.isdir(out_figure_path):
os.mkdir(out_figure_path)
png_append = '_' + img_name[0:img_name.find('.')] + '.png'
print('Output directory: ' + out_figure_path)
# DM corr matrix
cm = df[df.columns[0:-1]].corr()
curr_sz = copy.deepcopy(def_img_size)
if cm.shape[0] > def_img_size:
curr_sz = curr_sz + ((cm.shape[0] - curr_sz) * .3)
mtx_scale = curr_sz * 100
mask = np.zeros_like(cm, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
fig, ax = plt.subplots(figsize=(curr_sz, curr_sz))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(cm, mask=mask, cmap=cmap, center=0, vmax=cm[cm < 1].max().max(), vmin=cm[cm < 1].min().min(),
square=True, linewidths=.5, cbar_kws={"shrink": .6})
ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize)
ax.set_yticklabels(cm.columns.tolist(), rotation=-30, va='bottom', fontsize=fontsize)
ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title)
plt.tight_layout()
file_corr_matrix = 'Corr_matrix_regressors' + png_append
fig.savefig(pjoin(out_figure_path, file_corr_matrix))
plt.close(fig)
del fig, ax
# DM of Nuisance Regressors (all)
tr_label = 'TR (Volume #)'
fig, ax = plt.subplots(figsize=(curr_sz - 4.1, def_img_size))
x_scale_html = ((curr_sz - 4.1) / def_img_size) * 890
reporting.plot_design_matrix(df, ax=ax)
ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title)
ax.set_xticklabels(ax.get_xticklabels(), rotation=60, ha='right', fontsize=fontsize)
ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
ax.set_ylabel(tr_label, fontsize=fontsize)
plt.tight_layout()
file_design_matrix = 'Design_matrix' + png_append
fig.savefig(pjoin(out_figure_path, file_design_matrix))
plt.close(fig)
del fig, ax
# FD timeseries plot
FD = 'FD'
poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd']
fd_idx = [df_orig.columns.__contains__(i) for i in poss_names]
if np.sum(fd_idx) > 0:
FD_name = poss_names[fd_idx == True]
if sum(df_orig[FD_name].isnull()) > 0:
df_orig[FD_name] = df_orig[FD_name].fillna(np.mean(df_orig[FD_name]))
y = df_orig[FD_name].as_matrix()
Nremove = []
sc_idx = []
for thr_idx, thr in enumerate(FD_thr):
idx = y >= thr
sc_idx.append(copy.deepcopy(idx))
for iidx in np.where(idx)[0]:
for buffer in sc_range:
curr_idx = iidx + buffer
if curr_idx >= 0 and curr_idx <= len(idx):
sc_idx[thr_idx][curr_idx] = True
Nremove.append(np.sum(sc_idx[thr_idx]))
Nplots = len(FD_thr)
sns.set(font_scale=1.5)
sns.set_style('ticks')
fig, axes = plt.subplots(Nplots, 1, figsize=(def_img_size * 1.5, def_img_size / 2), squeeze=False)
sns.despine()
bound = .4
fd_mean = np.mean(y)
for curr in np.arange(0, Nplots):
axes[curr, 0].plot(y)
axes[curr, 0].plot((-bound, Ntrs + bound), FD_thr[curr] * np.ones((1, 2))[0], '--', color='black')
axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20)
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
axes[curr, 0].axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red')
axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')')
axes[curr, 0].set_title(FD + ': ' + str(100 * Nremove[curr] / Ntrs)[0:4]
+ '% of scan (' + str(Nremove[curr]) + ' volumes) would be scrubbed (FD thr.= ' +
str(FD_thr[curr]) + ')')
plt.text(Ntrs + 1, FD_thr[curr] - .01, FD + ' = ' + str(FD_thr[curr]), fontsize=fontsize)
plt.text(Ntrs, fd_mean - .01, 'avg = ' + str(fd_mean), fontsize=fontsize)
axes[curr, 0].set_xlim((-bound, Ntrs + 8))
plt.tight_layout()
axes[curr, 0].set_xlabel(tr_label)
file_fd_plot = FD + '_timeseries' + png_append
fig.savefig(pjoin(out_figure_path, file_fd_plot))
plt.close(fig)
del fig, axes
print(FD + ' timeseries plot saved')
else:
print(FD + ' not found: ' + FD + ' timeseries not plotted')
file_fd_plot = None
# Carpet and DVARS plots - before & after nuisance regression
# need to create mask file to input to DVARS function
mask_file = pjoin(out_figure_path, 'mask_temp.nii.gz')
nifti_masker = NiftiMasker(mask_strategy='epi', standardize=False)
nifti_masker.fit(img)
nifti_masker.mask_img_.to_filename(mask_file)
# create 2 or 3 carpet plots, depending on if LP filter is also applied
Ncarpet = 2
total_sz = int(16)
carpet_scale = 840
y_labels = ['Input (voxels)', 'Output \'cleaned\'']
imgs = [img, new_img]
img_files = [img_file, save_img_file]
color = ['red', 'salmon']
labels = ['input', 'cleaned']
if lp_filter:
Ncarpet = 3
total_sz = int(20)
carpet_scale = carpet_scale * (9/8)
y_labels = ['Input', 'Clean Pre-LP', 'Clean LP']
imgs.insert(1, temp_img)
img_files.insert(1, temp_img_file)
color.insert(1, 'firebrick')
labels.insert(1, 'clean pre-LP')
labels[-1] = 'clean LP'
dvars = []
print('Computing dvars...')
for in_file in img_files:
temp = nac.compute_dvars(in_file=in_file, in_mask=mask_file)[1]
dvars.append(np.hstack((temp.mean(), temp)))
del temp
small_sz = 2
fig = plt.figure(figsize=(def_img_size * 1.5, def_img_size + ((Ncarpet - 2) * 1)))
row_used = 0
if np.sum(fd_idx) > 0: # if FD data is available
row_used = row_used + small_sz
ax0 = plt.subplot2grid((total_sz, 1), (0, 0), rowspan=small_sz)
ax0.plot(y)
ax0.scatter(np.arange(0, Ntrs), y, s=10)
curr = 0
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
ax0.axvspan(temp.min() - bound, temp.max() + bound, alpha=.5, color='red')
ax0.set_ylabel(FD)
for side in ["top", "right", "bottom"]:
ax0.spines[side].set_color('none')
ax0.spines[side].set_visible(False)
ax0.set_xticks([])
ax0.set_xlim((-.5, Ntrs - .5))
ax0.spines["left"].set_position(('outward', 10))
ax_d = plt.subplot2grid((total_sz, 1), (row_used, 0), rowspan=small_sz)
for iplot in np.arange(len(dvars)):
ax_d.plot(dvars[iplot], color=color[iplot], label=labels[iplot])
ax_d.set_ylabel('DVARS')
for side in ["top", "right", "bottom"]:
ax_d.spines[side].set_color('none')
ax_d.spines[side].set_visible(False)
ax_d.set_xticks([])
ax_d.set_xlim((-.5, Ntrs - .5))
ax_d.spines["left"].set_position(('outward', 10))
ax_d.legend(fontsize=fontsize - 2)
row_used = row_used + small_sz
st = 0
carpet_each = int((total_sz - row_used) / Ncarpet)
for idx, img_curr in enumerate(imgs):
ax_curr = plt.subplot2grid((total_sz, 1), (row_used + st, 0), rowspan=carpet_each)
fig = plotting.plot_carpet(img_curr, figure=fig, axes=ax_curr)
ax_curr.set_ylabel(y_labels[idx])
for side in ["bottom", "left"]:
ax_curr.spines[side].set_position(('outward', 10))
if idx < len(imgs)-1:
ax_curr.spines["bottom"].set_visible(False)
ax_curr.set_xticklabels('')
ax_curr.set_xlabel('')
st = st + carpet_each
file_carpet_plot = 'Carpet_plots' + png_append
fig.savefig(pjoin(out_figure_path, file_carpet_plot))
plt.close()
del fig, ax0, ax_curr, ax_d, dvars
os.remove(mask_file)
print('Carpet/DVARS plots saved')
if lp_filter:
os.remove(temp_img_file)
del temp_img
# Display T-stat maps for nuisance regressors
# create mean img
img_size = (img.shape[0], img.shape[1], img.shape[2])
mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine)
mx = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
mx.append(np.max(np.absolute([con.t.min(), con.t.max()])))
mx = .8 * np.max(mx)
t_png = 'Tstat_'
file_tstat = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine)
title_str = col + ' Tstat'
fig = plotting.plot_stat_map(m_img, mean_img, threshold=3, colorbar=True, display_mode='z', vmax=mx,
title=title_str,
cut_coords=7)
file_temp = t_png + col + png_append
fig.savefig(pjoin(out_figure_path, file_temp))
file_tstat.append({'name': col, 'file': file_temp})
plt.close()
del fig, file_temp
print(title_str + ' map saved')
# Display R-sq map for nuisance regressors
m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine)
title_str = 'Nuisance Rsq'
mx = .95 * rsquare.max()
fig = plotting.plot_stat_map(m_img, mean_img, threshold=.2, colorbar=True, display_mode='z', vmax=mx,
title=title_str,
cut_coords=7)
file_rsq_map = 'Rsquared' + png_append
fig.savefig(pjoin(out_figure_path, file_rsq_map))
plt.close()
del fig
print(title_str + ' map saved')
######### html report
templateLoader = jinja2.FileSystemLoader(searchpath="/")
templateEnv = jinja2.Environment(loader=templateLoader)
templateVars = {"img_file": img_file,
"save_img_file": save_img_file,
"Ntrs": Ntrs,
"tsv_file": tsv_file,
"col_names": col_names,
"hp_filter": hp_filter,
"lp_filter": lp_filter,
"file_design_matrix": file_design_matrix,
"file_corr_matrix": file_corr_matrix,
"file_fd_plot": file_fd_plot,
"file_rsq_map": file_rsq_map,
"file_tstat": file_tstat,
"x_scale": x_scale_html,
"mtx_scale": mtx_scale,
"file_carpet_plot": file_carpet_plot,
"carpet_scale": carpet_scale
}
TEMPLATE_FILE = pjoin(os.getcwd(), "report_template.html")
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(templateVars)
html_file = pjoin(out_figure_path, img_name[0:img_name.find('.')] + '.html')
with open(html_file, "w") as f:
f.write(outputText)
print('')
print('HTML report: ' + html_file)
return new_img
if (hdr_input.nSamples - 1) < endsample:
print "Error: buffer reset detected"
raise SystemExit
endsample = hdr_input.nSamples - 1
if endsample < window:
continue
begsample = endsample - window + 1
D = ftc.getData([begsample, endsample])
D = D[:, chanindx]
if low_pass or high_pass:
D = signal.butterworth(D,
hdr_input.fSample,
low_pass=low_pass,
high_pass=high_pass,
order=order)
rms = []
for i in range(0, len(chanindx)):
rms.append(0)
for i, chanvec in enumerate(D.transpose()):
for chanval in chanvec:
rms[i] += chanval * chanval
rms[i] = math.sqrt(rms[i])
if debug > 1:
print rms
def _run_interface(self, runtime):
import os
import numpy as np
import pandas as pd
import SimpleITK as sitk
import nilearn.image
import nibabel as nb
from nilearn.signal import butterworth
from scipy.signal import detrend
from rabies.confound_correction_pkg.utils import recover_3D,recover_4D,temporal_censoring,lombscargle_fill, exec_ICA_AROMA,butterworth
from rabies.analysis_pkg.analysis_functions import closed_form
### set null returns in case the workflow is interrupted
empty_img = sitk.GetImageFromArray(np.empty([1,1]))
empty_file = os.path.abspath('empty.nii.gz')
sitk.WriteImage(empty_img, empty_file)
setattr(self, 'cleaned_path', empty_file)
setattr(self, 'VE_file_path', empty_file)
setattr(self, 'STD_file_path', empty_file)
setattr(self, 'CR_STD_file_path', empty_file)
setattr(self, 'frame_mask_file', empty_file)
setattr(self, 'data_dict', empty_file)
setattr(self, 'aroma_out', empty_file)
###
bold_file = self.inputs.bold_file
brain_mask_file = self.inputs.brain_mask_file
CSF_mask_file = self.inputs.CSF_mask_file
data_dict = self.inputs.data_dict
cr_opts = self.inputs.cr_opts
FD_trace=data_dict['FD_trace']
confounds_array=data_dict['confounds_array']
confounds_file=data_dict['confounds_csv']
time_range=data_dict['time_range']
confounds_6rigid_array=data_dict['confounds_6rigid_array']
cr_out = os.getcwd()
import pathlib # Better path manipulation
filename_split = pathlib.Path(bold_file).name.rsplit(".nii")
brain_mask = sitk.GetArrayFromImage(sitk.ReadImage(brain_mask_file, sitk.sitkFloat32))
volume_indices = brain_mask.astype(bool)
data_img = sitk.ReadImage(bold_file, sitk.sitkFloat32)
data_array = sitk.GetArrayFromImage(data_img)
num_volumes = data_array.shape[0]
timeseries = np.zeros([num_volumes, volume_indices.sum()])
for i in range(num_volumes):
timeseries[i, :] = (data_array[i, :, :, :])[volume_indices]
timeseries = timeseries[time_range,:]
if cr_opts.TR=='auto':
TR = float(data_img.GetSpacing()[3])
else:
TR = float(cr_opts.TR)
'''
#1 - Compute and apply frame censoring mask (from FD and/or DVARS thresholds)
'''
frame_mask,FD_trace,DVARS = temporal_censoring(timeseries, FD_trace,
cr_opts.FD_censoring, cr_opts.FD_threshold, cr_opts.DVARS_censoring, cr_opts.minimum_timepoint)
if frame_mask is None:
return runtime
timeseries = timeseries[frame_mask]
confounds_array = confounds_array[frame_mask]
'''
#2 - Linear detrending of fMRI timeseries and nuisance regressors
'''
# apply simple detrending, after censoring
timeseries = detrend(timeseries,axis=0)
confounds_array = detrend(confounds_array,axis=0) # apply detrending to the confounds too
'''
#3 - Apply ICA-AROMA.
'''
if cr_opts.run_aroma:
# write intermediary output files for timeseries and 6 rigid body parameters
timeseries_3d = recover_4D(brain_mask_file, timeseries, bold_file)
inFile = f'{cr_out}/{filename_split[0]}_aroma_input.nii.gz'
sitk.WriteImage(timeseries_3d, inFile)
confounds_6rigid_array=confounds_6rigid_array[frame_mask,:]
confounds_6rigid_array = detrend(confounds_6rigid_array,axis=0) # apply detrending to the confounds too
df = pd.DataFrame(confounds_6rigid_array)
df.columns = ['mov1', 'mov2', 'mov3', 'rot1', 'rot2', 'rot3']
mc_file = f'{cr_out}/{filename_split[0]}_aroma_input.csv'
df.to_csv(mc_file)
cleaned_file, aroma_out = exec_ICA_AROMA(inFile, mc_file, brain_mask_file, CSF_mask_file, TR, cr_opts.aroma_dim, random_seed=cr_opts.aroma_random_seed)
# if AROMA failed, returns empty outputs
if cleaned_file is None:
return runtime
setattr(self, 'aroma_out', aroma_out)
data_img = sitk.ReadImage(cleaned_file, sitk.sitkFloat32)
data_array = sitk.GetArrayFromImage(data_img)
num_volumes = data_array.shape[0]
timeseries = np.zeros([num_volumes, volume_indices.sum()])
for i in range(num_volumes):
timeseries[i, :] = (data_array[i, :, :, :])[volume_indices]
if (not cr_opts.highpass is None) or (not cr_opts.lowpass is None):
'''
#4 - If frequency filtering and frame censoring are applied, simulate data in censored timepoints using the Lomb-Scargle periodogram,
as suggested in Power et al. (2014, Neuroimage), for both the fMRI timeseries and nuisance regressors prior to filtering.
'''
timeseries_filled = lombscargle_fill(x=timeseries,time_step=TR,time_mask=frame_mask)
confounds_filled = lombscargle_fill(x=confounds_array,time_step=TR,time_mask=frame_mask)
'''
#5 - As recommended in Lindquist et al. (2019, Human brain mapping), make the nuisance regressors orthogonal
to the temporal filter.
'''
confounds_filtered = butterworth(confounds_filled, TR=TR,
high_pass=cr_opts.highpass, low_pass=cr_opts.lowpass)
'''
#6 - Apply highpass and/or lowpass filtering on the fMRI timeseries (with simulated timepoints).
'''
timeseries_filtered = butterworth(timeseries_filled, TR=TR,
high_pass=cr_opts.highpass, low_pass=cr_opts.lowpass)
# correct for edge effects of the filters
num_cut = int(cr_opts.edge_cutoff/TR)
if len(frame_mask)<2*num_cut:
raise ValueError(f"The timeseries are too short to remove {cr_opts.edge_cutoff}sec of data at each edge.")
if not num_cut==0:
frame_mask[:num_cut]=0
frame_mask[-num_cut:]=0
'''
#7 - Re-apply the frame censoring mask onto filtered fMRI timeseries and nuisance regressors, taking out the
simulated timepoints. Edge artefacts from frequency filtering can also be removed as recommended in Power et al. (2014, Neuroimage).
'''
# re-apply the masks to take out simulated data points, and take off the edges
timeseries = timeseries_filtered[frame_mask]
confounds_array = confounds_filtered[frame_mask]
if frame_mask.sum()<int(cr_opts.minimum_timepoint):
from nipype import logging
log = logging.getLogger('nipype.workflow')
log.warning(f"CONFOUND CORRECTION LEFT LESS THAN {str(cr_opts.minimum_timepoint)} VOLUMES. THIS SCAN WILL BE REMOVED FROM FURTHER PROCESSING.")
return runtime
'''
#8 - Apply confound regression using the selected nuisance regressors.
'''
# voxels that have a NaN value are set to 0
nan_voxels = np.isnan(timeseries).sum(axis=0)>1
timeseries[:,nan_voxels] = 0
# estimate the VE from the CR selection, or 6 rigid motion parameters if no CR is applied
X=confounds_array
Y=timeseries
try:
predicted = X.dot(closed_form(X,Y))
res = Y-predicted
except:
from nipype import logging
log = logging.getLogger('nipype.workflow')
log.warning("SINGULAR MATRIX ERROR DURING CONFOUND REGRESSION. THIS SCAN WILL BE REMOVED FROM FURTHER PROCESSING.")
empty_img = sitk.GetImageFromArray(np.empty([1,1]))
empty_file = os.path.abspath('empty.nii.gz')
sitk.WriteImage(empty_img, empty_file)
return runtime
# derive features from the predicted timeseries
predicted_std = predicted.std(axis=0)
predicted_time = np.sqrt((predicted.T**2).mean(axis=0))
VE_spatial = 1-(res.var(axis=0)/Y.var(axis=0))
VE_temporal = 1-(res.var(axis=1)/Y.var(axis=1))
if len(cr_opts.conf_list) > 0:
# if confound regression is applied
timeseries = res
# save the temporal STD map prior to standardization and smoothing
temporal_std = timeseries.std(axis=0)
'''
#8 - Standardize timeseries
'''
if cr_opts.standardize:
timeseries = (timeseries-timeseries.mean(axis=0))/temporal_std
# save output files
VE_spatial_map = recover_3D(brain_mask_file, VE_spatial)
STD_spatial_map = recover_3D(brain_mask_file, temporal_std)
CR_STD_spatial_map = recover_3D(brain_mask_file, predicted_std)
timeseries_3d = recover_4D(brain_mask_file, timeseries, bold_file)
cleaned_path = cr_out+'/'+filename_split[0]+'_cleaned.nii.gz'
sitk.WriteImage(timeseries_3d, cleaned_path)
VE_file_path = cr_out+'/'+filename_split[0]+'_VE_map.nii.gz'
sitk.WriteImage(VE_spatial_map, VE_file_path)
STD_file_path = cr_out+'/'+filename_split[0]+'_STD_map.nii.gz'
sitk.WriteImage(STD_spatial_map, STD_file_path)
CR_STD_file_path = cr_out+'/'+filename_split[0]+'_CR_STD_map.nii.gz'
sitk.WriteImage(CR_STD_spatial_map, CR_STD_file_path)
frame_mask_file = cr_out+'/'+filename_split[0]+'_frame_censoring_mask.csv'
pd.DataFrame(frame_mask).to_csv(frame_mask_file, index=False, header=['False = Masked Frames'])
if cr_opts.smoothing_filter is not None:
'''
#9 - Apply Gaussian spatial smoothing.
'''
timeseries_3d = nilearn.image.smooth_img(nb.load(cleaned_path), cr_opts.smoothing_filter)
timeseries_3d.to_filename(cleaned_path)
# apply the frame mask to FD trace/DVARS
DVARS = DVARS[frame_mask]
FD_trace = FD_trace[frame_mask]
# calculate temporal degrees of freedom left after confound correction
num_timepoints = frame_mask.sum()
if cr_opts.run_aroma:
aroma_rm = (pd.read_csv(f'{aroma_out}/classification_overview.txt', sep='\t')['Motion/noise']).sum()
else:
aroma_rm = 0
num_regressors = confounds_array.shape[1]
tDOF = num_timepoints - (aroma_rm+num_regressors)
data_dict = {'FD_trace':FD_trace, 'DVARS':DVARS, 'time_range':time_range, 'frame_mask':frame_mask, 'confounds_array':confounds_array, 'VE_temporal':VE_temporal, 'confounds_csv':confounds_file, 'predicted_time':predicted_time, 'tDOF':tDOF}
setattr(self, 'cleaned_path', cleaned_path)
setattr(self, 'VE_file_path', VE_file_path)
setattr(self, 'STD_file_path', STD_file_path)
setattr(self, 'CR_STD_file_path', CR_STD_file_path)
setattr(self, 'frame_mask_file', frame_mask_file)
setattr(self, 'data_dict', data_dict)
return runtime
def denoise(img_file,
tsv_file,
out_path,
col_names=False,
hp_filter=False,
lp_filter=False,
out_figure_path=False):
nii_ext = '.nii.gz'
FD_thr = [.5]
sc_range = np.arange(-1, 3)
constant = 'constant'
# get file info
base_file = os.path.basename(img_file)
save_img_file = pjoin(out_path, base_file[0:base_file.find('.')] + \
'_NR' + nii_ext)
# read in files
img = load_niimg(img_file)
data = img.get_data()
df = pandas.read_csv(tsv_file, '\t', na_values='n/a')
Ntrs = df.as_matrix().shape[0]
print('# of TRs: ' + str(Ntrs))
assert (Ntrs == data.shape[len(data.shape) - 1])
# select columns to use as nuisance regressors
str_append = ' [ALL regressors in CSV]'
if col_names:
df = df[col_names]
str_append = ' [SELECTED regressors in CSV]'
# fill in missing nuisance values with mean for that variable
for col in df.columns:
if sum(df[col].isnull()) > 0:
print('Filling in ' + str(sum(df[col].isnull())) +
' NaN value for ' + col)
df[col] = df[col].fillna(np.mean(df[col]))
print('# of Confound Regressors: ' + str(len(df.columns)) + str_append)
# implement HP filter in regression
TR = img.header.get_zooms()[-1]
frame_times = np.arange(Ntrs) * TR
if hp_filter:
hp_filter = float(hp_filter)
assert (hp_filter > 0)
period_cutoff = 1. / hp_filter
df = make_design_matrix(frame_times,
period_cut=period_cutoff,
add_regs=df.as_matrix(),
add_reg_names=df.columns.tolist())
# fn adds intercept into dm
hp_cols = [col for col in df.columns if 'drift' in col]
print('# of High-pass Filter Regressors: ' + str(len(hp_cols)))
else:
# add in intercept column into data frame
df[constant] = 1
dm = df.as_matrix()
# prep data
data = np.reshape(data, (-1, Ntrs))
data_mean = np.mean(data, axis=1)
Nvox = len(data_mean)
# setup and run regression
model = regression.OLSModel(dm)
results = model.fit(data.T)
if not hp_filter:
results_orig_resid = copy.deepcopy(
results.resid) # save for rsquared computation
# apply low-pass filter
if lp_filter:
# input to butterworth fn is time x voxels
low_pass = float(lp_filter)
Fs = 1. / TR
if low_pass >= Fs / 2:
raise ValueError(
'Low pass filter cutoff if too close to the Nyquist frequency (%s)'
% (Fs / 2))
results.resid = butterworth(results.resid,
sampling_rate=Fs,
low_pass=low_pass,
high_pass=None)
# add mean back into data
clean_data = results.resid.T + np.reshape(
data_mean, (Nvox, 1)) # add mean back into residuals
# save out new data file
clean_data = np.reshape(clean_data, img.shape).astype('float32')
#new_img = nb.Nifti1Image(clean_data, img.affine)
#inherit header from original
new_header = header = img.header.copy()
new_img = nb.Nifti1Image(clean_data, None, header=new_header)
new_img.to_filename(save_img_file)
######### generate Rsquared map for confounds only
if hp_filter:
# first remove low-frequency information from data
hp_cols.append(constant)
model_first = regression.OLSModel(df[hp_cols].as_matrix())
results_first = model_first.fit(data.T)
results_first_resid = copy.deepcopy(results_first.resid)
del results_first, model_first
# compute sst - borrowed from matlab
sst = np.square(
np.linalg.norm(results_first_resid -
np.mean(results_first_resid, axis=0),
axis=0))
# now regress out 'true' confounds to estimate their Rsquared
nr_cols = [col for col in df.columns if 'drift' not in col]
model_second = regression.OLSModel(df[nr_cols].as_matrix())
results_second = model_second.fit(results_first_resid)
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_second.resid, axis=0))
del results_second, model_second, results_first_resid
elif not hp_filter:
# compute sst - borrowed from matlab
sst = np.square(
np.linalg.norm(data.T - np.mean(data.T, axis=0), axis=0))
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_orig_resid, axis=0))
del results_orig_resid
# compute rsquared of nuisance regressors
zero_idx = scipy.logical_and(sst == 0, sse == 0)
sse[zero_idx] = 1
sst[zero_idx] = 1 # would be NaNs - become rsquared = 0
rsquare = 1 - np.true_divide(sse, sst)
rsquare[np.isnan(rsquare)] = 0
######### Visualizing DM & outputs
fontsize = 12
fontsize_title = 14
if not out_figure_path:
out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures'
if not os.path.isdir(out_figure_path):
os.mkdir(out_figure_path)
img_name = os.path.basename(img_file)
png_append = '_' + img_name[0:img_name.find('.')] + '.png'
# DM corr matrix
cm = df[df.columns[0:-1]].corr()
mask = np.zeros_like(cm, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
sz = 8
if cm.shape[0] > sz:
sz = sz + ((cm.shape[0] - sz) * .3)
fig, ax = plt.subplots(figsize=(sz, sz))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(cm,
mask=mask,
cmap=cmap,
center=0,
vmax=cm[cm < 1].max().max(),
vmin=cm[cm < 1].min().min(),
square=True,
linewidths=.5,
cbar_kws={"shrink": .6})
ax.set_xticklabels(ax.get_xticklabels(),
rotation=60,
ha='right',
fontsize=fontsize)
ax.set_yticklabels(cm.columns.tolist(),
rotation=-30,
va='bottom',
fontsize=fontsize)
ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title)
plt.tight_layout()
fig.savefig(pjoin(out_figure_path, 'Corr_matrix_regressors' + png_append))
plt.close(fig)
del fig, ax
# DM of Nuisance Regressors (all)
tr_label = 'TR (Volume #)'
fig, ax = plt.subplots(figsize=(4, sz))
reporting.plot_design_matrix(df, ax=ax)
ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title)
ax.set_xticklabels(ax.get_xticklabels(),
rotation=60,
ha='right',
fontsize=fontsize)
ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
ax.set_ylabel(tr_label, fontsize=fontsize)
plt.tight_layout()
fig.savefig(pjoin(out_figure_path, 'Design_matrix' + png_append))
plt.close(fig)
del fig, ax
# FD timeseries plot
FD = 'FD'
poss_names = ['FramewiseDisplacement', FD, 'framewisedisplacement', 'fd']
idx = [df.columns.__contains__(i) for i in poss_names]
FD_name = poss_names[idx == True]
y = df[FD_name].as_matrix()
Nremove = []
sc_idx = []
for thr_idx, thr in enumerate(FD_thr):
idx = y >= thr
sc_idx.append(copy.deepcopy(idx))
for iidx in np.where(idx)[0]:
for buffer in sc_range:
curr_idx = iidx + buffer
if curr_idx >= 0 and curr_idx <= len(idx):
sc_idx[thr_idx][curr_idx] = True
Nremove.append(np.sum(sc_idx[thr_idx]))
Nplots = len(FD_thr)
sns.set(font_scale=1.5)
sns.set_style('ticks')
fig, axes = plt.subplots(Nplots, 1, figsize=(12, 4), squeeze=False)
sns.despine()
bound = .4
for curr in np.arange(0, Nplots):
axes[curr, 0].plot(y)
axes[curr, 0].plot((-bound, Ntrs + bound),
FD_thr[curr] * np.ones((1, 2))[0],
'--',
color='black')
axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20)
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
axes[curr, 0].axvspan(temp.min() - bound,
temp.max() + bound,
alpha=.5,
color='red')
axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')')
axes[curr, 0].set_title(FD + ': ' +
str(100 * Nremove[curr] / Ntrs)[0:4] +
'% of scan (' + str(Nremove[curr]) +
' volumes) would be scrubbed (FD thr.= ' +
str(FD_thr[curr]) + ')')
plt.text(Ntrs + 1,
FD_thr[curr] - .01,
FD + ' = ' + str(FD_thr[curr]),
fontsize=fontsize)
axes[curr, 0].set_xlim((-bound, Ntrs + 8))
plt.tight_layout()
axes[curr, 0].set_xlabel(tr_label)
fig.savefig(pjoin(out_figure_path, FD + '_timeseries' + png_append))
plt.close(fig)
del fig, axes
# Display T-stat maps for nuisance regressors
# create mean img
img_size = (img.shape[0], img.shape[1], img.shape[2])
mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine)
mx = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
mx.append(np.max(np.absolute([con.t.min(), con.t.max()])))
mx = .8 * np.max(mx)
t_png = 'Tstat_'
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
print(con_vector)
m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine)
title_str = col + ' Tstat map '
print(title_str)
fig = plotting.plot_stat_map(m_img,
mean_img,
threshold=3,
colorbar=True,
display_mode='z',
vmax=mx,
title=title_str,
cut_coords=7)
fig.savefig(pjoin(out_figure_path, t_png + col + png_append))
plt.close()
del fig
# Display R-sq map for nuisance regressors
m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine)
title_str = 'Nuisance Rsq map '
print(title_str)
mx = .95 * rsquare.max()
fig = plotting.plot_stat_map(m_img,
mean_img,
threshold=.2,
colorbar=True,
display_mode='z',
vmax=mx,
title=title_str,
cut_coords=7)
fig.savefig(pjoin(out_figure_path, 'Rsquared' + png_append))
plt.close()
del fig
def test_butterworth():
rand_gen = np.random.RandomState(0)
n_features = 20000
n_samples = 100
sampling = 100
low_pass = 30
high_pass = 10
# Compare output for different options.
# single timeseries
data = rand_gen.randn(n_samples)
data_original = data.copy()
'''
May be only on py3.5:
Bug in scipy 1.1.0 generates an unavoidable FutureWarning.
(More info: https://github.com/scipy/scipy/issues/9086)
The number of warnings generated is overwhelming TravisCI's log limit,
causing it to fail tests.
This hack prevents that and will be removed in future.
'''
buggy_scipy = (LooseVersion(scipy.__version__) < LooseVersion('1.2')
and LooseVersion(scipy.__version__) > LooseVersion('1.0')
)
if buggy_scipy:
warnings.simplefilter('ignore')
''' END HACK '''
out_single = nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=False)
np.testing.assert_almost_equal(out_single, data)
np.testing.assert_(id(out_single) != id(data))
# multiple timeseries
data = rand_gen.randn(n_samples, n_features)
data[:, 0] = data_original # set first timeseries to previous data
data_original = data.copy()
out1 = nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=True)
np.testing.assert_almost_equal(data, data_original)
np.testing.assert_(id(out1) != id(data_original))
# check that multiple- and single-timeseries filtering do the same thing.
np.testing.assert_almost_equal(out1[:, 0], out_single)
nisignal.butterworth(data, sampling,
low_pass=low_pass, high_pass=high_pass,
copy=False)
np.testing.assert_almost_equal(out1, data)
# Test nyquist frequency clipping, issue #482
out1 = nisignal.butterworth(data, sampling,
low_pass=50.,
copy=True)
out2 = nisignal.butterworth(data, sampling,
low_pass=80., # Greater than nyq frequency
copy=True)
np.testing.assert_almost_equal(out1, out2)
np.testing.assert_(id(out1) != id(out2))
