def test_oversampling():
events = basic_paradigm()
frame_times = np.linspace(0, 127, 128)
X1 = make_first_level_design_matrix(frame_times, events, drift_model=None)
X2 = make_first_level_design_matrix(frame_times,
events,
drift_model=None,
oversampling=50)
X3 = make_first_level_design_matrix(frame_times,
events,
drift_model=None,
oversampling=10)
# oversampling = 50 by default so X2 = X1, X3 \neq X1, X3 close to X2
assert_almost_equal(X1.values, X2.values)
assert_almost_equal(X2.values, X3.values, 0)
assert (np.linalg.norm(X2.values - X3.values) / np.linalg.norm(X2.values) >
1.e-4)
# fir model, oversampling is forced to 1
X4 = make_first_level_design_matrix(frame_times,
events,
hrf_model='fir',
drift_model=None,
fir_delays=range(0, 4),
oversampling=1)
X5 = make_first_level_design_matrix(frame_times,
events,
hrf_model='fir',
drift_model=None,
fir_delays=range(0, 4),
oversampling=10)
assert_almost_equal(X4.values, X5.values)
def test_first_level_design_creation():
# Test processing of FMRI inputs
with InTemporaryDirectory():
shapes = ((7, 8, 9, 10), )
mask, FUNCFILE, _ = write_fake_fmri_data_and_design(shapes)
FUNCFILE = FUNCFILE[0]
func_img = load(FUNCFILE)
# basic test based on basic_paradigm and glover hrf
t_r = 10.0
slice_time_ref = 0.
events = basic_paradigm()
model = FirstLevelModel(t_r,
slice_time_ref,
mask_img=mask,
drift_model='polynomial',
drift_order=3)
model = model.fit(func_img, events)
frame1, X1, names1 = check_design_matrix(model.design_matrices_[0])
# check design computation is identical
n_scans = get_data(func_img).shape[3]
start_time = slice_time_ref * t_r
end_time = (n_scans - 1 + slice_time_ref) * t_r
frame_times = np.linspace(start_time, end_time, n_scans)
design = make_first_level_design_matrix(frame_times,
events,
drift_model='polynomial',
drift_order=3)
frame2, X2, names2 = check_design_matrix(design)
assert_array_equal(frame1, frame2)
assert_array_equal(X1, X2)
assert_array_equal(names1, names2)
# Delete objects attached to files to avoid WindowsError when deleting
# temporary directory (in Windows)
del FUNCFILE, mask, model, func_img
def _make_dummy_contrasts_dmtx():
frame_times = np.linspace(0, 127 * 1., 128)
dmtx = make_first_level_design_matrix(frame_times,
drift_model='polynomial',
drift_order=3,
)
contrast = {'test': np.ones(4)}
return contrast, dmtx
def test_high_pass():
""" test that high-pass values lead to reasonable design matrices"""
n_frames = 128
tr = 2.0
frame_times = np.arange(0, tr * n_frames, tr)
X = make_first_level_design_matrix(frame_times,
drift_model='Cosine',
high_pass=1.)
assert X.shape[1] == n_frames
def test_design_matrix0():
# Test design matrix creation when no experimental paradigm is provided
tr = 1.0
frame_times = np.linspace(0, 127 * tr, 128)
_, X, names = check_design_matrix(
make_first_level_design_matrix(frame_times,
drift_model='polynomial',
drift_order=3))
assert len(names) == 4
x = np.linspace(-0.5, .5, 128)
assert_almost_equal(X[:, 0], x)
def test_design_matrix0d():
# test design matrix creation when regressors are provided manually
tr = 1.0
frame_times = np.linspace(0, 127 * tr, 128)
ax = np.random.randn(128, 4)
_, X, names = check_design_matrix(
make_first_level_design_matrix(frame_times,
drift_model='polynomial',
drift_order=3,
add_regs=ax))
assert len(names) == 8
assert X.shape[1] == 8
def test_design_matrix0c():
# test design matrix creation when regressors are provided manually
rng = np.random.RandomState(42)
tr = 1.0
frame_times = np.linspace(0, 127 * tr, 128)
ax = rng.standard_normal(size=(128, 4))
_, X, names = check_design_matrix(
make_first_level_design_matrix(frame_times,
drift_model='polynomial',
drift_order=3,
add_regs=ax))
assert_almost_equal(X[:, 0], ax[:, 0])
ax = rng.standard_normal(size=(127, 4))
with pytest.raises(
AssertionError,
match="Incorrect specification of additional regressors:."):
make_first_level_design_matrix(frame_times, add_regs=ax)
ax = rng.standard_normal(size=(128, 4))
with pytest.raises(
ValueError,
match="Incorrect number of additional regressor names."):
make_first_level_design_matrix(frame_times,
add_regs=ax,
add_reg_names='')
# with pandas Dataframe
axdf = pd.DataFrame(ax)
_, X1, names = check_design_matrix(
make_first_level_design_matrix(frame_times,
drift_model='polynomial',
drift_order=3,
add_regs=axdf))
assert_almost_equal(X1[:, 0], ax[:, 0])
assert_array_equal(names[:4], np.arange(4))
def test_show_design_matrix():
# test that the show code indeed (formally) runs
frame_times = np.linspace(0, 127 * 1., 128)
dmtx = make_first_level_design_matrix(
frame_times, drift_model='polynomial', drift_order=3)
ax = plot_design_matrix(dmtx)
assert (ax is not None)
with InTemporaryDirectory():
ax = plot_design_matrix(dmtx, output_file='dmtx.png')
assert os.path.exists('dmtx.png')
assert (ax is None)
plot_design_matrix(dmtx, output_file='dmtx.pdf')
assert os.path.exists('dmtx.pdf')
def test_design_matrix0c():
# test design matrix creation when regressors are provided manually
tr = 1.0
frame_times = np.linspace(0, 127 * tr, 128)
ax = np.random.randn(128, 4)
_, X, names = check_design_matrix(
make_first_level_design_matrix(frame_times,
drift_model='polynomial',
drift_order=3,
add_regs=ax))
assert_almost_equal(X[:, 0], ax[:, 0])
ax = np.random.randn(127, 4)
with pytest.raises(
AssertionError,
match="Incorrect specification of additional regressors:."):
make_first_level_design_matrix(frame_times, add_regs=ax)
ax = np.random.randn(128, 4)
with pytest.raises(
ValueError,
match="Incorrect number of additional regressor names."):
make_first_level_design_matrix(frame_times,
add_regs=ax,
add_reg_names='')
def test_spm_2():
# Check that the nistats design matrix is close enough to the SPM one
# (it cannot be identical, because the hrf shape is different)
frame_times = np.linspace(0, 99, 100)
conditions = ['c0', 'c0', 'c0', 'c1', 'c1', 'c1', 'c2', 'c2', 'c2']
onsets = [30, 50, 70, 10, 30, 80, 30, 40, 60]
durations = 10 * np.ones(9)
events = pd.DataFrame({
'trial_type': conditions,
'onset': onsets,
'duration': durations
})
X1 = make_first_level_design_matrix(frame_times, events, drift_model=None)
spm_design_matrix = DESIGN_MATRIX['arr_1']
_, matrix, _ = check_design_matrix(X1)
assert (((spm_design_matrix - matrix)**2).sum() /
(spm_design_matrix**2).sum() < .1)
def report_flm_adhd_dmn(): # pragma: no cover
t_r = 2.
slice_time_ref = 0.
n_scans = 176
pcc_coords = (0, -53, 26)
adhd_dataset = nilearn.datasets.fetch_adhd(n_subjects=1)
seed_masker = NiftiSpheresMasker([pcc_coords],
radius=10,
detrend=True,
standardize=True,
low_pass=0.1,
high_pass=0.01,
t_r=2.,
memory='nilearn_cache',
memory_level=1,
verbose=0)
seed_time_series = seed_masker.fit_transform(adhd_dataset.func[0])
frametimes = np.linspace(0, (n_scans - 1) * t_r, n_scans)
design_matrix = make_first_level_design_matrix(frametimes,
hrf_model='spm',
add_regs=seed_time_series,
add_reg_names=["pcc_seed"])
dmn_contrast = np.array([1] + [0] * (design_matrix.shape[1] - 1))
contrasts = {'seed_based_glm': dmn_contrast}
first_level_model = FirstLevelModel(t_r=t_r, slice_time_ref=slice_time_ref)
first_level_model = first_level_model.fit(run_imgs=adhd_dataset.func[0],
design_matrices=design_matrix)
report = make_glm_report(
first_level_model,
contrasts=contrasts,
title='ADHD DMN Report',
cluster_threshold=15,
height_control='bonferroni',
min_distance=8.,
plot_type='glass',
report_dims=(1200, 'a'),
)
output_filename = 'generated_report_flm_adhd_dmn.html'
output_filepath = os.path.join(REPORTS_DIR, output_filename)
report.save_as_html(output_filepath)
report.get_iframe()
def test_csv_io():
# test the csv io on design matrices
tr = 1.0
frame_times = np.linspace(0, 127 * tr, 128)
events = modulated_event_paradigm()
DM = make_first_level_design_matrix(
frame_times,
events,
hrf_model='glover',
drift_model='polynomial',
drift_order=3,
)
path = 'design_matrix.csv'
with InTemporaryDirectory():
DM.to_csv(path)
DM2 = pd.read_csv(path, index_col=0)
_, matrix, names = check_design_matrix(DM)
_, matrix_, names_ = check_design_matrix(DM2)
assert_almost_equal(matrix, matrix_)
assert names == names_
def design_matrix_light(frame_times,
events=None,
hrf_model='glover',
drift_model='cosine',
high_pass=.01,
drift_order=1,
fir_delays=None,
add_regs=None,
add_reg_names=None,
min_onset=-24,
path=None):
""" Same as make_first_level_design_matrix,
but only returns the computed matrix and associated name.
"""
fir_delays = fir_delays if fir_delays else [0]
dmtx = make_first_level_design_matrix(frame_times, events, hrf_model,
drift_model, high_pass, drift_order,
fir_delays, add_regs, add_reg_names,
min_onset)
_, matrix, names = check_design_matrix(dmtx)
return matrix, names
def _run_interface(self, runtime):
import nibabel as nb
from nilearn.glm.first_level import design_matrix as dm
info = self.inputs.session_info
img = nb.load(self.inputs.bold_file)
if isinstance(img, nb.Cifti2Image):
vols = img.shape[0]
elif isinstance(img, nb.Nifti1Image):
vols = img.shape[3]
elif isinstance(img, nb.GiftiImage):
vols = len(img.darrays)
else:
raise ValueError(
f"Unknown image type ({img.__class__.__name__}) <{self.inputs.bold_file}>"
)
drop_missing = bool(self.inputs.drop_missing)
drift_model = self.inputs.drift_model
if info['sparse'] not in (None, 'None'):
sparse = pd.read_hdf(info['sparse'],
key='sparse').rename(columns={
'condition': 'trial_type',
'amplitude': 'modulation'
})
if 'modulation' in sparse.columns:
sparse = sparse.dropna(subset=['modulation']) # Drop NAs
else:
sparse = None
if info['dense'] not in (None, 'None'):
dense = pd.read_hdf(info['dense'], key='dense')
missing_columns = dense.isna().all()
if drop_missing:
# Remove columns with NaNs
dense = dense[dense.columns[~missing_columns]]
elif missing_columns.any():
missing_names = ', '.join(
dense.columns[missing_columns].tolist())
raise RuntimeError(
f'The following columns are empty: {missing_names}. '
'Use --drop-missing to drop before model fitting.')
column_names = dense.columns.tolist()
drift_model = None if (('cosine00' in column_names) |
('cosine_00' in column_names)) else \
drift_model
if dense.empty:
dense = None
column_names = None
else:
dense = None
column_names = None
mat = dm.make_first_level_design_matrix(
frame_times=np.arange(vols) * info['repetition_time'],
events=sparse,
add_regs=dense,
hrf_model=None, # XXX: Consider making an input spec parameter
add_reg_names=column_names,
drift_model=drift_model)
mat.to_csv('design.tsv', sep='\t')
self._results['design_matrix'] = os.path.join(runtime.cwd,
'design.tsv')
return runtime
def create_design_matrix(tr,
frame_times,
events,
hrf_model='kay',
hrf_idx=None):
""" Creates a design matrix based on a HRF from Kendrick Kay's set
or a default one from Nilearn. """
# This is to keep oversampling consistent across hrf_models
hrf_oversampling = 10
design_oversampling = tr / (0.1 / hrf_oversampling)
if hrf_model != 'kay': # just use Nilearn!
return make_first_level_design_matrix(frame_times,
events,
drift_model=None,
min_onset=0,
oversampling=design_oversampling,
hrf_model=hrf_model)
if hrf_model == 'kay':
if hrf_idx is None: # 20 different DMs (based on different HRFs)
to_iter = range(HRFS_HR.shape[1])
else: # use the supplied HRF idx (e.g., 5)
to_iter = [hrf_idx]
dms = [] # will store all design matrices
for hrf_idx in to_iter: # iterate across all HRFs
hrf = HRFS_HR[:, hrf_idx]
# scale HRF to have the same max as the glover HRF
# makes comparison easier
hrf /= (hrf.max() / 0.249007)
# Get info
trial_type, onset, duration, modulation = check_events(events)
# Pre-allocate design matrix; note: columns are alphabetically sorted
X = np.zeros((frame_times.size, np.unique(trial_type).size))
uniq_trial_types = np.unique(trial_type) # this is sorted
# Create separate regressor for each unique trial type
# Code copied from Nilearn glm module
for i, condition in enumerate(uniq_trial_types):
condition_mask = (trial_type == condition)
exp_condition = (onset[condition_mask],
duration[condition_mask],
modulation[condition_mask])
# Create high resolution regressor/frame times
hr_regressor, hr_frame_times = _sample_condition(
exp_condition, frame_times, design_oversampling, 0)
# Convolve with HRF and downsample
conv_reg = np.convolve(hr_regressor, hrf)[:hr_regressor.size]
# linear interpolation for now ...
f = interp1d(hr_frame_times, conv_reg)
X[:, i] = f(frame_times).T
# Note to self: do not scale such that max(X, axis=0) is 1, because you'll lose info
# about predictor variance!
dm = pd.DataFrame(X, columns=uniq_trial_types, index=frame_times)
dm['constant'] = 1 # and intercept/constant
dms.append(dm)
if len(dms) == 1:
# Just return single design matrix
dms = dms[0]
return dms
fd = open(sd.func[0].split(".")[0] + "_onset.txt", "w")
for c, o, d in zip(conditions, onset, duration):
fd.write("%s %s %s\r\n" % (c, o, d))
fd.close()
# preprocess the data
subject_data = do_subjects_preproc(jobfile, dataset_dir=dataset_dir)[0]
# construct design matrix
nscans = len(subject_data.func[0])
frametimes = np.linspace(0, (nscans - 1) * tr, nscans)
drift_model = 'Cosine'
hrf_model = 'spm + derivative'
design_matrix = make_first_level_design_matrix(frametimes,
paradigm,
hrf_model=hrf_model,
drift_model=drift_model,
high_pass=hfcut)
# plot and save design matrix
ax = plot_design_matrix(design_matrix)
ax.set_position([.05, .25, .9, .65])
ax.set_title('Design matrix')
dmat_outfile = os.path.join(subject_data.output_dir, 'design_matrix.png')
plt.savefig(dmat_outfile, bbox_inches="tight", dpi=200)
# specify contrasts
contrasts = {}
_, matrix, names = check_design_matrix(design_matrix)
contrast_matrix = np.eye(len(names))
for i in range(len(names)):
def first_level(subject_dic,
additional_regressors=None,
compcorr=False,
smooth=None,
mesh=False,
mask_img=None):
""" Run the first-level analysis (GLM fitting + statistical maps)
in a given subject
Parameters
----------
subject_dic: dict,
exhaustive description of an individual acquisition
additional_regressors: dict or None,
additional regressors provided as an already sampled
design_matrix
dictionary keys are session_ids
compcorr: Bool, optional,
whether confound estimation and removal should be done or not
smooth: float or None, optional,
how much the data should spatially smoothed during masking
"""
start_time = time.ctime()
# experimental paradigm meta-params
motion_names = ['tx', 'ty', 'tz', 'rx', 'ry', 'rz']
hrf_model = subject_dic['hrf_model']
high_pass = subject_dic['high_pass']
drift_model = subject_dic['drift_model']
tr = subject_dic['TR']
slice_time_ref = 1.
if not mesh and (mask_img is None):
mask_img = masking(subject_dic['func'], subject_dic['output_dir'])
if additional_regressors is None:
additional_regressors = dict([
(session_id, None) for session_id in subject_dic['session_id']
])
for session_id, fmri_path, onset, motion_path in zip(
subject_dic['session_id'], subject_dic['func'],
subject_dic['onset'], subject_dic['realignment_parameters']):
task_id = _session_id_to_task_id([session_id])[0]
if mesh is not False:
from nibabel.gifti import read
n_scans = np.array(
[darrays.data for darrays in read(fmri_path).darrays]).shape[0]
else:
n_scans = nib.load(fmri_path).shape[3]
# motion parameters
motion = np.loadtxt(motion_path)
# define the time stamps for different images
frametimes = np.linspace(slice_time_ref,
(n_scans - 1 + slice_time_ref) * tr, n_scans)
if task_id == 'audio':
mask = np.array([1, 0, 1, 1, 0, 1, 1, 0, 1, 1])
n_cycles = 28
cycle_duration = 20
t_r = 2
cycle = np.arange(0, cycle_duration, t_r)[mask > 0]
frametimes = np.tile(cycle, n_cycles) +\
np.repeat(np.arange(n_cycles) * cycle_duration, mask.sum())
frametimes = frametimes[:-2] # for some reason...
if mesh is not False:
compcorr = False # XXX Fixme
if compcorr:
confounds = high_variance_confounds(fmri_path, mask_img=mask_img)
confounds = np.hstack((confounds, motion))
confound_names = ['conf_%d' % i for i in range(5)] + motion_names
else:
confounds = motion
confound_names = motion_names
if onset is None:
warnings.warn('Onset file not provided. Trying to guess it')
task = os.path.basename(fmri_path).split('task')[-1][4:]
onset = os.path.join(
os.path.split(os.path.dirname(fmri_path))[0], 'model001',
'onsets', 'task' + task + '_run001', 'task%s.csv' % task)
if not os.path.exists(onset):
warnings.warn('non-existant onset file. proceeding without it')
paradigm = None
else:
paradigm = make_paradigm(onset, task_id)
# handle manually supplied regressors
add_reg_names = []
if additional_regressors[session_id] is None:
add_regs = confounds
else:
df = read_csv(additional_regressors[session_id])
add_regs = []
for regressor in df:
add_reg_names.append(regressor)
add_regs.append(df[regressor])
add_regs = np.array(add_regs).T
add_regs = np.hstack((add_regs, confounds))
add_reg_names += confound_names
# create the design matrix
design_matrix = make_first_level_design_matrix(
frametimes,
paradigm,
hrf_model=hrf_model,
drift_model=drift_model,
high_pass=high_pass,
add_regs=add_regs,
add_reg_names=add_reg_names)
_, dmtx, names = check_design_matrix(design_matrix)
# create the relevant contrasts
contrasts = make_contrasts(task_id, names)
if mesh == 'fsaverage5':
# this is low-resolution data
subject_session_output_dir = os.path.join(
subject_dic['output_dir'], 'res_fsaverage5_%s' % session_id)
elif mesh == 'fsaverage7':
subject_session_output_dir = os.path.join(
subject_dic['output_dir'], 'res_fsaverage7_%s' % session_id)
elif mesh == 'individual':
subject_session_output_dir = os.path.join(
subject_dic['output_dir'], 'res_individual_%s' % session_id)
else:
subject_session_output_dir = os.path.join(
subject_dic['output_dir'], 'res_stats_%s' % session_id)
if not os.path.exists(subject_session_output_dir):
os.makedirs(subject_session_output_dir)
np.savez(os.path.join(subject_session_output_dir, 'design_matrix.npz'),
design_matrix=design_matrix)
if mesh is not False:
run_surface_glm(design_matrix, contrasts, fmri_path,
subject_session_output_dir)
else:
z_maps, fmri_glm = run_glm(design_matrix,
contrasts,
fmri_path,
mask_img,
subject_dic,
subject_session_output_dir,
tr=tr,
slice_time_ref=slice_time_ref,
smoothing_fwhm=smooth)
# do stats report
anat_img = nib.load(subject_dic['anat'])
stats_report_filename = os.path.join(subject_session_output_dir,
'report_stats.html')
report = make_glm_report(
fmri_glm,
contrasts,
threshold=3.0,
bg_img=anat_img,
cluster_threshold=15,
title="GLM for subject %s" % session_id,
)
report.save_as_html(stats_report_filename)
def _run_interface(self, runtime):
import nibabel as nb
from nilearn.glm.first_level import design_matrix as dm
info = self.inputs.design_info
img = nb.load(self.inputs.bold_file)
if isinstance(img, nb.Cifti2Image):
vols = img.shape[0]
elif isinstance(img, nb.Nifti1Image):
vols = img.shape[3]
elif isinstance(img, nb.GiftiImage):
vols = len(img.darrays)
else:
raise ValueError(
f"Unknown image type ({img.__class__.__name__}) <{self.inputs.bold_file}>"
)
drop_missing = bool(self.inputs.drop_missing)
drift_model = self.inputs.drift_model
if info['dense'] not in (None, 'None'):
dense = pd.read_hdf(info['dense'], key='dense')
missing_columns = dense.isna().all()
if drop_missing:
# Remove columns with NaNs
dense = dense[dense.columns[~missing_columns]]
elif missing_columns.any():
missing_names = ', '.join(
dense.columns[missing_columns].tolist())
raise RuntimeError(
f'The following columns are empty: {missing_names}. '
'Use --drop-missing to drop before model fitting.')
column_names = dense.columns.tolist()
drift_model = (None if
(('cosine00' in column_names) |
('cosine_00' in column_names)) else drift_model)
if dense.empty:
dense = None
column_names = None
else:
dense = None
column_names = None
# XXX hack for pybids giving us a bad matrix
# Case seems to be when (TR * nvols) rounds up instead of down
if len(dense) == vols + 1 and np.isnan(dense[-1:].values).all():
dense = dense[:-1]
mat = dm.make_first_level_design_matrix(
frame_times=np.arange(vols) * info['repetition_time'],
add_regs=dense,
hrf_model=None, # XXX: Consider making an input spec parameter
add_reg_names=column_names,
drift_model=drift_model,
)
mat.to_csv('design.tsv', sep='\t')
self._results['design_matrix'] = os.path.join(runtime.cwd,
'design.tsv')
return runtime
def denoise(img_file,
tsv_file,
out_path,
col_names=False,
hp_filter=False,
lp_filter=False,
out_figure_path=False):
nii_ext = '.nii.gz'
FD_thr = [.5]
sc_range = np.arange(-1, 3)
constant = 'constant'
# read in files
img = load_niimg(img_file)
# get file info
img_name = os.path.basename(img.get_filename())
file_base = img_name[0:img_name.find('.')]
save_img_file = pjoin(out_path, file_base + \
'_NR' + nii_ext)
data = img.get_fdata()
df_orig = pandas.read_csv(tsv_file, '\t', na_values='n/a')
df = copy.deepcopy(df_orig)
Ntrs = df.values.shape[0]
print('# of TRs: ' + str(Ntrs))
assert (Ntrs == data.shape[len(data.shape) - 1])
# select columns to use as nuisance regressors
if col_names:
df = df[col_names]
str_append = ' [SELECTED regressors in CSV]'
else:
col_names = df.columns.tolist()
str_append = ' [ALL regressors in CSV]'
# fill in missing nuisance values with mean for that variable
for col in df.columns:
if sum(df[col].isnull()) > 0:
print('Filling in ' + str(sum(df[col].isnull())) +
' NaN value for ' + col)
df[col] = df[col].fillna(np.mean(df[col]))
print('# of Confound Regressors: ' + str(len(df.columns)) + str_append)
# implement HP filter in regression
TR = img.header.get_zooms()[-1]
frame_times = np.arange(Ntrs) * TR
if hp_filter:
hp_filter = float(hp_filter)
assert (hp_filter > 0)
df = make_first_level_design_matrix(frame_times,
high_pass=hp_filter,
add_regs=df.values,
add_reg_names=df.columns.tolist())
# fn adds intercept into dm
hp_cols = [col for col in df.columns if 'drift' in col]
print('# of High-pass Filter Regressors: ' + str(len(hp_cols)))
else:
# add in intercept column into data frame
df[constant] = 1
print('No High-pass Filter Applied')
dm = df.values
# prep data
data = np.reshape(data, (-1, Ntrs))
data_mean = np.mean(data, axis=1)
Nvox = len(data_mean)
# setup and run regression
model = regression.OLSModel(dm)
results = model.fit(data.T)
if not hp_filter:
results_orig_resid = copy.deepcopy(
results.residuals) # save for rsquared computation
# apply low-pass filter
if lp_filter:
# input to butterworth fn is time x voxels
low_pass = float(lp_filter)
Fs = 1. / TR
if low_pass >= Fs / 2:
raise ValueError(
'Low pass filter cutoff if too close to the Nyquist frequency (%s)'
% (Fs / 2))
temp_img_file = pjoin(out_path, file_base + \
'_temp' + nii_ext)
temp_img = nb.Nifti1Image(np.reshape(
results.residuals.T + np.reshape(data_mean, (Nvox, 1)),
img.shape).astype('float32'),
img.affine,
header=img.header)
temp_img.to_filename(temp_img_file)
results.residuals = butterworth(results.residuals,
sampling_rate=Fs,
low_pass=low_pass,
high_pass=None)
print('Low-pass Filter Applied: < ' + str(low_pass) + ' Hz')
# add mean back into data
clean_data = results.residuals.T + np.reshape(
data_mean, (Nvox, 1)) # add mean back into residuals
# save out new data file
print('Saving output file...')
clean_data = np.reshape(clean_data, img.shape).astype('float32')
new_img = nb.Nifti1Image(clean_data, img.affine, header=img.header)
new_img.to_filename(save_img_file)
######### generate Rsquared map for confounds only
if hp_filter:
# first remove low-frequency information from data
hp_cols.append(constant)
model_first = regression.OLSModel(df[hp_cols].values)
results_first = model_first.fit(data.T)
results_first_resid = copy.deepcopy(results_first.residuals)
del results_first, model_first
# compute sst - borrowed from matlab
sst = np.square(
np.linalg.norm(results_first_resid -
np.mean(results_first_resid, axis=0),
axis=0))
# now regress out 'true' confounds to estimate their Rsquared
nr_cols = [col for col in df.columns if 'drift' not in col]
model_second = regression.OLSModel(df[nr_cols].values)
results_second = model_second.fit(results_first_resid)
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_second.residuals, axis=0))
del results_second, model_second, results_first_resid
elif not hp_filter:
# compute sst - borrowed from matlab
sst = np.square(
np.linalg.norm(data.T - np.mean(data.T, axis=0), axis=0))
# compute sse - borrowed from matlab
sse = np.square(np.linalg.norm(results_orig_resid, axis=0))
del results_orig_resid
# compute rsquared of nuisance regressors
zero_idx = np.logical_and(sst == 0, sse == 0)
sse[zero_idx] = 1
sst[zero_idx] = 1 # would be NaNs - become rsquared = 0
rsquare = 1 - np.true_divide(sse, sst)
rsquare[np.isnan(rsquare)] = 0
######### Visualizing DM & outputs
fontsize = 12
fontsize_title = 14
def_img_size = 8
if not out_figure_path:
out_figure_path = save_img_file[0:save_img_file.find('.')] + '_figures'
if not os.path.isdir(out_figure_path):
os.mkdir(out_figure_path)
png_append = '_' + img_name[0:img_name.find('.')] + '.png'
print('Output directory: ' + out_figure_path)
# DM corr matrix
cm = df[df.columns[0:-1]].corr()
curr_sz = copy.deepcopy(def_img_size)
if cm.shape[0] > def_img_size:
curr_sz = curr_sz + ((cm.shape[0] - curr_sz) * .3)
mtx_scale = curr_sz * 100
mask = np.zeros_like(cm, dtype=np.bool)
mask[np.triu_indices_from(mask)] = True
fig, ax = plt.subplots(figsize=(curr_sz, curr_sz))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(cm,
mask=mask,
cmap=cmap,
center=0,
vmax=cm[cm < 1].max().max(),
vmin=cm[cm < 1].min().min(),
square=True,
linewidths=.5,
cbar_kws={"shrink": .6})
ax.set_xticklabels(ax.get_xticklabels(),
rotation=60,
ha='right',
fontsize=fontsize)
ax.set_yticklabels(cm.columns.tolist(),
rotation=-30,
va='bottom',
fontsize=fontsize)
ax.set_title('Nuisance Corr. Matrix', fontsize=fontsize_title)
plt.tight_layout()
file_corr_matrix = 'Corr_matrix_regressors' + png_append
fig.savefig(pjoin(out_figure_path, file_corr_matrix))
plt.close(fig)
del fig, ax
# DM of Nuisance Regressors (all)
tr_label = 'TR (Volume #)'
fig, ax = plt.subplots(figsize=(curr_sz - 4.1, def_img_size))
x_scale_html = ((curr_sz - 4.1) / def_img_size) * 890
plotting.plot_design_matrix(df, ax=ax)
ax.set_title('Nuisance Design Matrix', fontsize=fontsize_title)
ax.set_xticklabels(ax.get_xticklabels(),
rotation=60,
ha='right',
fontsize=fontsize)
ax.set_yticklabels(ax.get_yticklabels(), fontsize=fontsize)
ax.set_ylabel(tr_label, fontsize=fontsize)
plt.tight_layout()
file_design_matrix = 'Design_matrix' + png_append
fig.savefig(pjoin(out_figure_path, file_design_matrix))
plt.close(fig)
del fig, ax
# FD timeseries plot
FD = 'FD'
poss_names = [
'FramewiseDisplacement', FD, 'framewisedisplacement', 'fd',
'framewise_displacement'
]
fd_idx = [df_orig.columns.__contains__(i) for i in poss_names]
if np.sum(fd_idx) > 0:
FD_name = np.array(poss_names)[fd_idx][0] #poss_names[fd_idx == True]
if sum(df_orig[FD_name].isnull()) > 0:
df_orig[FD_name] = df_orig[FD_name].fillna(
np.mean(df_orig[FD_name]))
y = df_orig[FD_name].values
Nremove = []
sc_idx = []
for thr_idx, thr in enumerate(FD_thr):
idx = y >= thr
sc_idx.append(copy.deepcopy(idx))
for iidx in np.where(idx)[0]:
for buffer in sc_range:
curr_idx = iidx + buffer
if curr_idx >= 0 and curr_idx <= len(idx):
sc_idx[thr_idx][curr_idx] = True
Nremove.append(np.sum(sc_idx[thr_idx]))
Nplots = len(FD_thr)
sns.set(font_scale=1.5)
sns.set_style('ticks')
fig, axes = plt.subplots(Nplots,
1,
figsize=(def_img_size * 1.5,
def_img_size / 2),
squeeze=False)
sns.despine()
bound = .4
fd_mean = np.mean(y)
for curr in np.arange(0, Nplots):
axes[curr, 0].plot(y)
axes[curr, 0].plot((-bound, Ntrs + bound),
FD_thr[curr] * np.ones((1, 2))[0],
'--',
color='black')
axes[curr, 0].scatter(np.arange(0, Ntrs), y, s=20)
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
axes[curr, 0].axvspan(temp.min() - bound,
temp.max() + bound,
alpha=.5,
color='red')
axes[curr, 0].set_ylabel('Framewise Disp. (' + FD + ')')
axes[curr, 0].set_title(FD + ': ' +
str(100 * Nremove[curr] / Ntrs)[0:4] +
'% of scan (' + str(Nremove[curr]) +
' volumes) would be scrubbed (FD thr.= ' +
str(FD_thr[curr]) + ')')
plt.text(Ntrs + 1,
FD_thr[curr] - .01,
FD + ' = ' + str(FD_thr[curr]),
fontsize=fontsize)
plt.text(Ntrs,
fd_mean - .01,
'avg = ' + str(fd_mean),
fontsize=fontsize)
axes[curr, 0].set_xlim((-bound, Ntrs + 8))
plt.tight_layout()
axes[curr, 0].set_xlabel(tr_label)
file_fd_plot = FD + '_timeseries' + png_append
fig.savefig(pjoin(out_figure_path, file_fd_plot))
plt.close(fig)
del fig, axes
print(FD + ' timeseries plot saved')
else:
print(FD + ' not found: ' + FD + ' timeseries not plotted')
file_fd_plot = None
# Carpet and DVARS plots - before & after nuisance regression
# need to create mask file to input to DVARS function
mask_file = pjoin(out_figure_path, 'mask_temp.nii.gz')
nifti_masker = NiftiMasker(mask_strategy='epi', standardize=False)
nifti_masker.fit(img)
nifti_masker.mask_img_.to_filename(mask_file)
# create 2 or 3 carpet plots, depending on if LP filter is also applied
Ncarpet = 2
total_sz = int(16)
carpet_scale = 840
y_labels = ['Input (voxels)', 'Output \'cleaned\'']
imgs = [img, new_img]
img_files = [img_file, save_img_file]
color = ['red', 'salmon']
labels = ['input', 'cleaned']
if lp_filter:
Ncarpet = 3
total_sz = int(20)
carpet_scale = carpet_scale * (9 / 8)
y_labels = ['Input', 'Clean Pre-LP', 'Clean LP']
imgs.insert(1, temp_img)
img_files.insert(1, temp_img_file)
color.insert(1, 'firebrick')
labels.insert(1, 'clean pre-LP')
labels[-1] = 'clean LP'
dvars = []
print('Computing dvars...')
for in_file in img_files:
temp = nac.compute_dvars(in_file=in_file, in_mask=mask_file)[1]
dvars.append(np.hstack((temp.mean(), temp)))
del temp
small_sz = 2
fig = plt.figure(figsize=(def_img_size * 1.5,
def_img_size + ((Ncarpet - 2) * 1)))
row_used = 0
if np.sum(fd_idx) > 0: # if FD data is available
row_used = row_used + small_sz
ax0 = plt.subplot2grid((total_sz, 1), (0, 0), rowspan=small_sz)
ax0.plot(y)
ax0.scatter(np.arange(0, Ntrs), y, s=10)
curr = 0
if Nremove[curr] > 0:
info = scipy.ndimage.measurements.label(sc_idx[curr])
for cluster in np.arange(1, info[1] + 1):
temp = np.where(info[0] == cluster)[0]
ax0.axvspan(temp.min() - bound,
temp.max() + bound,
alpha=.5,
color='red')
ax0.set_ylabel(FD)
for side in ["top", "right", "bottom"]:
ax0.spines[side].set_color('none')
ax0.spines[side].set_visible(False)
ax0.set_xticks([])
ax0.set_xlim((-.5, Ntrs - .5))
ax0.spines["left"].set_position(('outward', 10))
ax_d = plt.subplot2grid((total_sz, 1), (row_used, 0), rowspan=small_sz)
for iplot in np.arange(len(dvars)):
ax_d.plot(dvars[iplot], color=color[iplot], label=labels[iplot])
ax_d.set_ylabel('DVARS')
for side in ["top", "right", "bottom"]:
ax_d.spines[side].set_color('none')
ax_d.spines[side].set_visible(False)
ax_d.set_xticks([])
ax_d.set_xlim((-.5, Ntrs - .5))
ax_d.spines["left"].set_position(('outward', 10))
ax_d.legend(fontsize=fontsize - 2)
row_used = row_used + small_sz
st = 0
carpet_each = int((total_sz - row_used) / Ncarpet)
for idx, img_curr in enumerate(imgs):
ax_curr = plt.subplot2grid((total_sz, 1), (row_used + st, 0),
rowspan=carpet_each)
fig = plotting.plot_carpet(img_curr, figure=fig, axes=ax_curr)
ax_curr.set_ylabel(y_labels[idx])
for side in ["bottom", "left"]:
ax_curr.spines[side].set_position(('outward', 10))
if idx < len(imgs) - 1:
ax_curr.spines["bottom"].set_visible(False)
ax_curr.set_xticklabels('')
ax_curr.set_xlabel('')
st = st + carpet_each
file_carpet_plot = 'Carpet_plots' + png_append
fig.savefig(pjoin(out_figure_path, file_carpet_plot))
plt.close()
del fig, ax0, ax_curr, ax_d, dvars
os.remove(mask_file)
print('Carpet/DVARS plots saved')
if lp_filter:
os.remove(temp_img_file)
del temp_img
# Display T-stat maps for nuisance regressors
# create mean img
img_size = (img.shape[0], img.shape[1], img.shape[2])
mean_img = nb.Nifti1Image(np.reshape(data_mean, img_size), img.affine)
mx = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
mx.append(np.max(np.absolute([con.t.min(), con.t.max()])))
mx = .8 * np.max(mx)
t_png = 'Tstat_'
file_tstat = []
for idx, col in enumerate(df.columns):
if not 'drift' in col and not constant in col:
con_vector = np.zeros((1, df.shape[1]))
con_vector[0, idx] = 1
con = results.Tcontrast(con_vector)
m_img = nb.Nifti1Image(np.reshape(con, img_size), img.affine)
title_str = col + ' Tstat'
fig = plotting.plot_stat_map(m_img,
mean_img,
threshold=3,
colorbar=True,
display_mode='z',
vmax=mx,
title=title_str,
cut_coords=7)
file_temp = t_png + col + png_append
fig.savefig(pjoin(out_figure_path, file_temp))
file_tstat.append({'name': col, 'file': file_temp})
plt.close()
del fig, file_temp
print(title_str + ' map saved')
# Display R-sq map for nuisance regressors
m_img = nb.Nifti1Image(np.reshape(rsquare, img_size), img.affine)
title_str = 'Nuisance Rsq'
mx = .95 * rsquare.max()
fig = plotting.plot_stat_map(m_img,
mean_img,
threshold=.2,
colorbar=True,
display_mode='z',
vmax=mx,
title=title_str,
cut_coords=7)
file_rsq_map = 'Rsquared' + png_append
fig.savefig(pjoin(out_figure_path, file_rsq_map))
plt.close()
del fig
print(title_str + ' map saved')
######### html report
templateLoader = jinja2.FileSystemLoader(searchpath="/")
templateEnv = jinja2.Environment(loader=templateLoader)
templateVars = {
"img_file": img_file,
"save_img_file": save_img_file,
"Ntrs": Ntrs,
"tsv_file": tsv_file,
"col_names": col_names,
"hp_filter": hp_filter,
"lp_filter": lp_filter,
"file_design_matrix": file_design_matrix,
"file_corr_matrix": file_corr_matrix,
"file_fd_plot": file_fd_plot,
"file_rsq_map": file_rsq_map,
"file_tstat": file_tstat,
"x_scale": x_scale_html,
"mtx_scale": mtx_scale,
"file_carpet_plot": file_carpet_plot,
"carpet_scale": carpet_scale
}
TEMPLATE_FILE = pjoin(os.getcwd(), "report_template.html")
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(templateVars)
html_file = pjoin(out_figure_path,
img_name[0:img_name.find('.')] + '.html')
with open(html_file, "w") as f:
f.write(outputText)
print('')
print('HTML report: ' + html_file)
return new_img
def fit(self, run_imgs, events=None, confounds=None,
design_matrices=None):
""" Fit the GLM
For each run:
1. create design matrix X
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
run_imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/manipulating_images/input_output.html#inputing-data-file-names-or-image-objects # noqa:E501
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
events: pandas Dataframe or string or list of pandas DataFrames or
strings
fMRI events used to build design matrices. One events object
expected per run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
confounds: pandas Dataframe, numpy array or string or
list of pandas DataFrames, numpy arays or strings
Each column in a DataFrame corresponds to a confound variable
to be included in the regression model of the respective run_img.
The number of rows must match the number of volumes in the
respective run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
design_matrices: pandas DataFrame or list of pandas DataFrames,
Design matrices that will be used to fit the GLM. If given it
takes precedence over events and confounds.
"""
# Local import to prevent circular imports
from nilearn.input_data import NiftiMasker # noqa
# Check arguments
# Check imgs type
if events is not None:
_check_events_file_uses_tab_separators(events_files=events)
if not isinstance(run_imgs, (list, tuple)):
run_imgs = [run_imgs]
if design_matrices is None:
if events is None:
raise ValueError('events or design matrices must be provided')
if self.t_r is None:
raise ValueError('t_r not given to FirstLevelModel object'
' to compute design from events')
else:
design_matrices = _check_run_tables(run_imgs, design_matrices,
'design_matrices')
# Check that number of events and confound files match number of runs
# Also check that events and confound files can be loaded as DataFrame
if events is not None:
events = _check_run_tables(run_imgs, events, 'events')
if confounds is not None:
confounds = _check_run_tables(run_imgs, confounds, 'confounds')
# Learn the mask
if self.mask_img is False:
# We create a dummy mask to preserve functionality of api
ref_img = check_niimg(run_imgs[0])
self.mask_img = Nifti1Image(np.ones(ref_img.shape[:3]),
ref_img.affine)
if not isinstance(self.mask_img, NiftiMasker):
self.masker_ = NiftiMasker(mask_img=self.mask_img,
smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize,
mask_strategy='epi',
t_r=self.t_r,
memory=self.memory,
verbose=max(0, self.verbose - 2),
target_shape=self.target_shape,
memory_level=self.memory_level
)
self.masker_.fit(run_imgs[0])
else:
if self.mask_img.mask_img_ is None and self.masker_ is None:
self.masker_ = clone(self.mask_img)
for param_name in ['target_affine', 'target_shape',
'smoothing_fwhm', 't_r', 'memory',
'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker'
' overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(run_imgs[0])
else:
self.masker_ = self.mask_img
# For each run fit the model and keep only the regression results.
self.labels_, self.results_, self.design_matrices_ = [], [], []
n_runs = len(run_imgs)
t0 = time.time()
for run_idx, run_img in enumerate(run_imgs):
# Report progress
if self.verbose > 0:
percent = float(run_idx) / n_runs
percent = round(percent * 100, 2)
dt = time.time() - t0
# We use a max to avoid a division by zero
if run_idx == 0:
remaining = 'go take a coffee, a big one'
else:
remaining = (100. - percent) / max(0.01, percent) * dt
remaining = '%i seconds remaining' % remaining
sys.stderr.write(
"Computing run %d out of %d runs (%s)\n"
% (run_idx + 1, n_runs, remaining))
# Build the experimental design for the glm
run_img = check_niimg(run_img, ensure_ndim=4)
if design_matrices is None:
n_scans = get_data(run_img).shape[3]
if confounds is not None:
confounds_matrix = confounds[run_idx].values
if confounds_matrix.shape[0] != n_scans:
raise ValueError('Rows in confounds does not match'
'n_scans in run_img at index %d'
% (run_idx,))
confounds_names = confounds[run_idx].columns.tolist()
else:
confounds_matrix = None
confounds_names = None
start_time = self.slice_time_ref * self.t_r
end_time = (n_scans - 1 + self.slice_time_ref) * self.t_r
frame_times = np.linspace(start_time, end_time, n_scans)
design = make_first_level_design_matrix(frame_times,
events[run_idx],
self.hrf_model,
self.drift_model,
self.high_pass,
self.drift_order,
self.fir_delays,
confounds_matrix,
confounds_names,
self.min_onset
)
else:
design = design_matrices[run_idx]
self.design_matrices_.append(design)
# Mask and prepare data for GLM
if self.verbose > 1:
t_masking = time.time()
sys.stderr.write('Starting masker computation \r')
Y = self.masker_.transform(run_img)
del run_img # Delete unmasked image to save memory
if self.verbose > 1:
t_masking = time.time() - t_masking
sys.stderr.write('Masker took %d seconds \n'
% t_masking)
if self.signal_scaling:
Y, _ = mean_scaling(Y, self.scaling_axis)
if self.memory:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
# compute GLM
if self.verbose > 1:
t_glm = time.time()
sys.stderr.write('Performing GLM computation\r')
labels, results = mem_glm(Y, design.values,
noise_model=self.noise_model,
bins=100, n_jobs=self.n_jobs)
if self.verbose > 1:
t_glm = time.time() - t_glm
sys.stderr.write('GLM took %d seconds \n' % t_glm)
self.labels_.append(labels)
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.results_.append(results)
del Y
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of %d runs done in %i seconds\n\n"
% (n_runs, time.time() - t0))
return self