def preprocess(num, subj, subj_dir, subj_warp_dir, force_warp=False):
bold_path = 'BOLD/task001_run00%i/bold_dico_bold7Tp1_to_subjbold7Tp1.nii.gz' % (num+1)
bold_path = os.path.join(DATA_DIR, subj, bold_path)
template_path = os.path.join(DATA_DIR, 'templates', 'grpbold7Tp1', 'brain.nii.gz')
warp_path = os.path.join(DATA_DIR, subj, 'templates', 'bold7Tp1', 'in_grpbold7Tp1', 'subj2tmpl_warp.nii.gz')
output_path = os.path.join(subj_warp_dir, 'run00%i.nii.gz' % num)
if force_warp or not os.path.exists(output_path):
print 'Warping image #%i...' % num
subprocess.call(['fsl5.0-applywarp', '-i', bold_path, '-o', output_path, '-r', template_path, '-w', warp_path, '-d', 'float'])
else:
print 'Reusing cached warp image #%i' % num
print 'Loading image #%i...' % num
bold = load(output_path)
masker = NiftiMasker(load(MASK_FILE))
# masker = niftimasker(load(MASK_FILE), detrend=true, smoothing_fwhm=4.0,
# high_pass=0.01, t_r=2.0, standardize=true)
masker.fit()
print 'Removing confounds from image #%i...' % num
data = masker.transform(bold, confounds(num, subj))
print 'Detrending image #%i...' % num
filtered = np.float32(savgol_filter(data, 61, 5, axis=0))
img = masker.inverse_transform(data-filtered)
print 'Smoothing image #%i...' % num
img = image.smooth_img(img, 4.0)
print 'Saving image #%i...' % num
save(img, os.path.join(subj_dir, 'run00%i.nii.gz' % num))
print 'Finished with image #%i' % num
def load_data():
with open(expanduser('~/data/HCP_unmasked/data.json'), 'r') as f:
data = json.load(f)
for this_data in data:
this_data['array'] += '.npy'
mask_img = expanduser('~/data/HCP_mask/mask_img.nii.gz')
masker = NiftiMasker(mask_img=mask_img, smoothing_fwhm=4,
standardize=True)
masker.fit()
smith2009 = fetch_atlas_smith_2009()
init = smith2009.rsn70
dict_init = masker.transform(init)
return masker, dict_init, sorted(data, key=lambda t: t['filename'])
class SmoothResampleMasker(BaseMasker):
def __init__(self, mask_img=None, smoothing_fwhm=None, resampling=None, searchlight=False):
self.mask_img = mask_img
self.smoothing_fwhm = smoothing_fwhm
self.resampling = resampling
self.searchlight = searchlight
self.masker = None
def fit(self):
if self.resampling is not None:
self.mask_img = resample_img(self.mask_img, target_affine=np.diag(self.resampling * np.ones(3)))
self.masker = NiftiMasker(mask_img=self.mask_img)
self.masker.fit()
return self
def transform(self, imgs, confounds=None):
smooth_prefix = '' if self.smoothing_fwhm is None else 's%g' % self.smoothing_fwhm
resample_prefix = '' if self.smoothing_fwhm is None else 'r%g' % self.smoothing_fwhm
if not isinstance(imgs, list):
imgs = [imgs]
path_first = imgs[0] if isinstance(imgs[0], str) else imgs[0].get_filename()
path_first_resampled = os.path.join(os.path.dirname(path_first), resample_prefix + os.path.basename(path_first))
path_first_smoothed = os.path.join(os.path.dirname(path_first), smooth_prefix + resample_prefix + os.path.basename(path_first))
if self.resampling is not None and self.smoothing_fwhm is not None:
if self.resampling is not None:
if not os.path.exists(path_first_resampled) and not os.path.exists(path_first_smoothed):
imgs = resample_img(imgs, target_affine=np.diag(self.resampling * np.ones(3)))
else:
imgs = []
if self.smoothing_fwhm is not None:
if not os.path.exists(path_first_smoothed):
imgs = smooth_img(imgs, self.smoothing_fwhm)
else:
imgs = []
else:
imgs = [check_niimg_3d(img) for img in imgs]
return self.masker.transform(imgs)
def test_multi_pca_score():
shape = (6, 8, 10, 5)
affine = np.eye(4)
rng = np.random.RandomState(0)
# Create a "multi-subject" dataset
imgs = []
for i in range(8):
this_img = rng.normal(size=shape)
imgs.append(nibabel.Nifti1Image(this_img, affine))
mask_img = nibabel.Nifti1Image(np.ones(shape[:3], dtype=np.int8), affine)
# Assert that score is between zero and one
multi_pca = MultiPCA(mask=mask_img, random_state=0, memory_level=0,
n_components=3)
multi_pca.fit(imgs)
s = multi_pca.score(imgs)
assert_true(np.all(s <= 1))
assert_true(np.all(0 <= s))
# Assert that score does not fail with single subject data
multi_pca = MultiPCA(mask=mask_img, random_state=0, memory_level=0,
n_components=3)
multi_pca.fit(imgs[0])
s = multi_pca.score(imgs[0])
assert_true(isinstance(s, float))
assert(0. <= s <= 1.)
# Assert that score is one for n_components == n_sample
# in single subject configuration
multi_pca = MultiPCA(mask=mask_img, random_state=0, memory_level=0,
n_components=5)
multi_pca.fit(imgs[0])
s = multi_pca.score(imgs[0])
assert_almost_equal(s, 1., 1)
# Per component score
multi_pca = MultiPCA(mask=mask_img, random_state=0, memory_level=0,
n_components=5)
multi_pca.fit(imgs[0])
masker = NiftiMasker(mask_img).fit()
s = multi_pca._raw_score(masker.transform(imgs[0]), per_component=True)
assert_equal(s.shape, (5,))
assert_true(np.all(s <= 1))
assert_true(np.all(0 <= s))
def get_init_objective(output_dir):
mask, func_filenames = get_hcp_data(raw=True)
masker = NiftiMasker(mask_img=mask, smoothing_fwhm=None,
standardize=False)
masker.fit()
rsn70 = fetch_atlas_smith_2009().rsn70
components = masker.transform(rsn70)
print(components.shape)
enet_scale(components.T, inplace=True)
print(np.sum(np.abs(components), axis=1))
test_data = func_filenames[(-n_test_records * 2)::2]
n_samples, n_voxels = np.load(test_data[-1], mmap_mode='r').shape
X = np.empty((n_test_records * n_samples, n_voxels))
for i, this_data in enumerate(test_data):
X[i * n_samples:(i + 1) * n_samples] = np.load(this_data,
mmap_mode='r')
exp_var = {}
for alpha in [1e-2, 1e-3, 1e-4]:
exp_var[alpha] = objective_function(X, components, alpha)
json.dump(open(join(output_dir, 'init_objective.json'), 'r'))
class SecondLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for multiple subject
fMRI data
Parameters
----------
mask_img: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters. Automatic mask computation assumes first level imgs have
already been masked.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints final computation time.
If 2 prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
"""
@replace_parameters({'mask': 'mask_img'}, end_version='next')
def __init__(self, mask_img=None, smoothing_fwhm=None,
memory=Memory(None), memory_level=1, verbose=0,
n_jobs=1, minimize_memory=True):
self.mask_img = mask_img
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, _basestring):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
self.second_level_input_ = None
self.confounds_ = None
def fit(self, second_level_input, confounds=None, design_matrix=None):
""" Fit the second-level GLM
1. create design matrix
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
second_level_input: list of `FirstLevelModel` objects or pandas
DataFrame or list of Niimg-like objects.
Giving FirstLevelModel objects will allow to easily compute
the second level contast of arbitrary first level contrasts thanks
to the first_level_contrast argument of the compute_contrast
method. Effect size images will be computed for each model to
contrast at the second level.
If a pandas DataFrame, then they have to contain subject_label,
map_name and effects_map_path. It can contain multiple maps that
would be selected during contrast estimation with the argument
first_level_contrast of the compute_contrast function. The
DataFrame will be sorted based on the subject_label column to avoid
order inconsistencies when extracting the maps. So the rows of the
automatically computed design matrix, if not provided, will
correspond to the sorted subject_label column.
If list of Niimg-like objects then this is taken literally as Y
for the model fit and design_matrix must be provided.
confounds: pandas DataFrame, optional
Must contain a subject_label column. All other columns are
considered as confounds and included in the model. If
design_matrix is provided then this argument is ignored.
The resulting second level design matrix uses the same column
names as in the given DataFrame for confounds. At least two columns
are expected, "subject_label" and at least one confound.
design_matrix: pandas DataFrame, optional
Design matrix to fit the GLM. The number of rows
in the design matrix must agree with the number of maps derived
from second_level_input.
Ensure that the order of maps given by a second_level_input
list of Niimgs matches the order of the rows in the design matrix.
"""
# check second_level_input
_check_second_level_input(second_level_input, design_matrix,
confounds=confounds)
# check confounds
_check_confounds(confounds)
# check design matrix
_check_design_matrix(design_matrix)
# sort a pandas dataframe by subject_label to avoid inconsistencies
# with the design matrix row order when automatically extracting maps
if isinstance(second_level_input, pd.DataFrame):
columns = second_level_input.columns.tolist()
column_index = columns.index('subject_label')
sorted_matrix = sorted(
second_level_input.values, key=lambda x: x[column_index])
sorted_input = pd.DataFrame(sorted_matrix, columns=columns)
second_level_input = sorted_input
self.second_level_input_ = second_level_input
self.confounds_ = confounds
# Report progress
t0 = time.time()
if self.verbose > 0:
sys.stderr.write("Fitting second level model. "
"Take a deep breath\r")
# Select sample map for masker fit and get subjects_label for design
if isinstance(second_level_input, pd.DataFrame):
sample_map = second_level_input['effects_map_path'][0]
labels = second_level_input['subject_label']
subjects_label = labels.values.tolist()
elif isinstance(second_level_input, Nifti1Image):
sample_map = mean_img(second_level_input)
elif isinstance(second_level_input[0], FirstLevelModel):
sample_model = second_level_input[0]
sample_condition = sample_model.design_matrices_[0].columns[0]
sample_map = sample_model.compute_contrast(
sample_condition, output_type='effect_size')
labels = [model.subject_label for model in second_level_input]
subjects_label = labels
else:
# In this case design matrix had to be provided
sample_map = mean_img(second_level_input)
# Create and set design matrix, if not given
if design_matrix is None:
design_matrix = make_second_level_design_matrix(subjects_label,
confounds)
self.design_matrix_ = design_matrix
# Learn the mask. Assume the first level imgs have been masked.
if not isinstance(self.mask_img, NiftiMasker):
self.masker_ = NiftiMasker(
mask_img=self.mask_img, smoothing_fwhm=self.smoothing_fwhm,
memory=self.memory, verbose=max(0, self.verbose - 1),
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask_img)
for param_name in ['smoothing_fwhm', 'memory', 'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(sample_map)
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of second level model done in "
"%i seconds\n" % (time.time() - t0))
return self
def compute_contrast(
self, second_level_contrast=None, first_level_contrast=None,
second_level_stat_type=None, output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
Parameters
----------
second_level_contrast: str or array of shape (n_col), optional
Where ``n_col`` is the number of columns of the design matrix,
The string can be a formula compatible with the linear constraint
of the Patsy library. Basically one can use the name of the
conditions as they appear in the design matrix of
the fitted model combined with operators /\*+- and numbers.
Please check the patsy documentation for formula examples:
http://patsy.readthedocs.io/en/latest/API-reference.html#patsy.DesignInfo.linear_constraint
The default (None) is accepted if the design matrix has a single
column, in which case the only possible contrast array([1]) is
applied; when the design matrix has multiple columns, an error is
raised.
first_level_contrast: str or array of shape (n_col) with respect to
FirstLevelModel, optional
In case a list of FirstLevelModel was provided as
second_level_input, we have to provide a contrast to apply to
the first level models to get the corresponding list of images
desired, that would be tested at the second level. In case a
pandas DataFrame was provided as second_level_input this is the
map name to extract from the pandas dataframe map_name column.
It has to be a 't' contrast.
second_level_stat_type: {'t', 'F'}, optional
Type of the second level contrast
output_type: str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size', 'effect_variance' or 'all'
Returns
-------
output_image: Nifti1Image
The desired output image(s). If ``output_type == 'all'``, then
the output is a dictionary of images, keyed by the type of image.
"""
if self.second_level_input_ is None:
raise ValueError('The model has not been fit yet')
# check first_level_contrast
_check_first_level_contrast(self.second_level_input_,
first_level_contrast)
# check contrast and obtain con_val
con_val = _get_con_val(second_level_contrast, self.design_matrix_)
# check output type
# 'all' is assumed to be the final entry;
# if adding more, place before 'all'
valid_types = ['z_score', 'stat', 'p_value', 'effect_size',
'effect_variance', 'all']
_check_output_type(output_type, valid_types)
# Get effect_maps appropriate for chosen contrast
effect_maps = _infer_effect_maps(self.second_level_input_,
first_level_contrast)
# Check design matrix X and effect maps Y agree on number of rows
_check_effect_maps(effect_maps, self.design_matrix_)
# Fit an Ordinary Least Squares regression for parametric statistics
Y = self.masker_.transform(effect_maps)
if self.memory:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
labels, results = mem_glm(Y, self.design_matrix_.values,
n_jobs=self.n_jobs, noise_model='ols')
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.labels_ = labels
self.results_ = results
# We compute contrast object
if self.memory:
mem_contrast = self.memory.cache(compute_contrast)
else:
mem_contrast = compute_contrast
contrast = mem_contrast(self.labels_, self.results_, con_val,
second_level_stat_type)
output_types = \
valid_types[:-1] if output_type == 'all' else [output_type]
outputs = {}
for output_type_ in output_types:
# We get desired output from contrast object
estimate_ = getattr(contrast, output_type_)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_val)
output.header['descrip'] = (
'%s of contrast %s' % (output_type, contrast_name))
outputs[output_type_] = output
return outputs if output_type == 'all' else output
labels = np.int32(labels)
# contrasts are IN ORDER -> shuffle!
new_inds = np.arange(0, X_task.shape[0])
np.random.shuffle(new_inds)
X_task = X_task[new_inds]
labels = labels[new_inds]
# subs = subs[new_inds]
# rest
# X_rest = nifti_masker.transform('preload_HR20persub_10mm_ero2.nii')
# X_rest = nifti_masker.transform('dump_rs_spca_s12_tmp')
rs_spca_data = joblib.load('dump_rs_spca_s12_tmp')
rs_spca_niis = nib.Nifti1Image(rs_spca_data,
nifti_masker.mask_img_.get_affine())
X_rest = nifti_masker.transform(rs_spca_niis)
del rs_spca_niis
del rs_spca_data
X_task = StandardScaler().fit_transform(X_task)
X_rest = StandardScaler().fit_transform(X_rest)
# ARCHI task
AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm')
AT_X = nifti_masker.transform(AT_niis)
AT_X = StandardScaler().fit_transform(AT_X)
print('done :)')
def make_new_noise(masker):
# CONST
'cos_ring_pos':
pjoin(work_dir, 'res_stats_exp_ring_pa', 'z_score_maps',
'cos.nii.gz'),
'sin_ring_pos':
pjoin(work_dir, 'res_stats_exp_ring_pa', 'z_score_maps',
'sin.nii.gz'),
'sin_ring_neg':
pjoin(work_dir, 'res_stats_cont_ring_ap', 'z_score_maps',
'sin.nii.gz'),
'cos_ring_neg':
pjoin(work_dir, 'res_stats_cont_ring_ap', 'z_score_maps',
'cos.nii.gz')
}
retino_coefs = {}
for key in retino_imgs.keys():
retino_coefs[key] = masker.transform(retino_imgs[key])
phase_wedge, phase_ring, phase_hemo = phase_maps(
retino_coefs,
offset_ring=np.pi,
offset_wedge=0.,
do_wedge=True,
do_ring=True,
)
phase_wedge_img = masker.inverse_transform(phase_wedge)
phase_ring_img = masker.inverse_transform(phase_ring)
phase_hemo_img = masker.inverse_transform(phase_hemo)
phase_wedge_img.to_filename(pjoin(write_dir, 'phase_wedge.nii.gz'))
phase_ring_img.to_filename(pjoin(write_dir, 'phase_ring.nii.gz'))
X_test = nibabel.Nifti1Image(niimgs.get_data()[:, :, :, condition_mask_test],
niimgs.get_affine())
y_test = target[condition_mask_test]
y_train[y_train == 'scissors'] = 1
y_train[y_train == 'scrambledpix'] = -1
y_train = np.array(y_train.astype('double'))
y_test[y_test == 'scissors'] = 1
y_test[y_test == 'scrambledpix'] = -1
y_test = np.array(y_test.astype('double'))
masker = NiftiMasker(mask_strategy='epi', standardize=True)
X_train = masker.fit_transform(X_train)
X_test = masker.transform(X_test)
mask = masker.mask_img_.get_data().astype(np.bool)
mask = _crop_mask(mask)
background_img = mean_img(data_files.func[0])
X_train, y_train, _, y_train_mean, _ = center_data(X_train,
y_train,
fit_intercept=True,
normalize=False,
copy=False)
X_test -= X_train.mean(axis=0)
X_test /= np.std(X_train, axis=0)
alpha = 1
ratio = 0.5
k = 200
# get HCP data
HCP_contrasts = [
'REWARD-PUNISH', 'PUNISH-REWARD', 'SHAPES-FACES', 'FACES-SHAPES',
'RANDOM-TOM', 'TOM-RANDOM', 'MATH-STORY', 'STORY-MATH', 'T-AVG', 'F-H',
'H-F', 'MATCH-REL', 'REL-MATCH', 'BODY-AVG', 'FACE-AVG', 'PLACE-AVG',
'TOOL-AVG', '2BK-0BK'
]
mean_supp = np.zeros((18, mask_nvox))
from nilearn.image import resample_img
for itask, task in enumerate(HCP_contrasts):
cur_nii = op.join(means_path, 'mean_%s.nii.gz' % (task))
print(cur_nii)
res_nii = resample_img(cur_nii,
target_affine=nifti_masker.mask_img_.get_affine(),
target_shape=nifti_masker.mask_img_.shape)
task_mean = nifti_masker.transform(res_nii)
mean_supp[itask, :] = task_mean
mean_supp_z = zscore(mean_supp, axis=1)
# get classification weights
lr_supp = np.load(op.join(LR_DIR, 'V0comps.npy'))
lr_supp_z = zscore(lr_supp, axis=1)
# get LR/AE weights
WRITE_DIR = 'nips3mm_recovery'
lambs = [0.25, 0.5, 0.75, 1]
import re
from scipy.stats import pearsonr
for n_comp in [5]:
corr_means_lr = np.zeros((len(lambs), 18))
corr_means_lr_ae = np.zeros((len(lambs), 18))
### Visualize the mask ########################################################
import matplotlib.pyplot as plt
import numpy as np
import nibabel
plt.figure()
plt.axis('off')
plt.imshow(np.rot90(nibabel.load(dataset.func[0]).get_data()[..., 20, 0]),
interpolation='nearest', cmap=plt.cm.gray)
ma = np.ma.masked_equal(mask, False)
plt.imshow(np.rot90(ma[..., 20]), interpolation='nearest', cmap=plt.cm.autumn,
alpha=0.5)
plt.title("Mask")
### Preprocess data ###########################################################
nifti_masker.fit(dataset.func[0])
fmri_masked = nifti_masker.transform(dataset.func[0])
### Run an algorithm ##########################################################
from sklearn.decomposition import FastICA
n_components = 20
ica = FastICA(n_components=n_components, random_state=42)
components_masked = ica.fit_transform(fmri_masked.T).T
### Reverse masking ###########################################################
components = nifti_masker.inverse_transform(components_masked)
### Show results ##############################################################
components_data = np.ma.masked_equal(components.get_data(), 0)
plt.figure()
plt.axis('off')
plt.imshow(np.rot90(nibabel.load(dataset.func[0]).get_data()[..., 20, 0]),
class FirstLevelGLM(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for Single-session fMRI data
Parameters
----------
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters.
target_affine: 3x3 or 4x4 matrix, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
target_shape: 3-tuple of integers, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
low_pass: False or float, optional
This parameter is passed to nilearn.signal.clean.
Please see the related documentation for details.
high_pass: False or float, optional
This parameter is passed to nilearn.signal.clean.
Please see the related documentation for details.
t_r: float, optional
This parameter is passed to nilearn.signal.clean.
Please see the related documentation for details.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: instance of joblib.Memory or string
Used to cache the masking process.
By default, no caching is done. If a string is given, it is the
path to the caching directory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
standardize : boolean, optional
If standardize is True, the time-series are centered and normed:
their variance is put to 1 in the time dimension.
percent_signal_change: bool, optional,
If True, fMRI signals are scaled to percent of the mean value
Incompatible with standardize (standardize=False is enforced when\
percent_signal_change is True).
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
noise_model : {'ar1', 'ols'}, optional
the temporal variance model. Defaults to 'ar1'
Attributes
----------
labels : array of shape (n_voxels,),
a map of values on voxels used to identify the corresponding model
results : dict,
with keys corresponding to the different labels values
values are RegressionResults instances corresponding to the voxels
"""
def __init__(self, mask=None, target_affine=None, target_shape=None,
low_pass=None, high_pass=None, t_r=None, smoothing_fwhm=None,
memory=Memory(None), memory_level=1, standardize=False,
percent_signal_change=True, verbose=1, n_jobs=1,
noise_model='ar1'):
self.mask = mask
self.memory = memory
self.memory_level = memory_level
self.verbose = verbose
self.standardize = standardize
self.n_jobs = n_jobs
self.low_pass = low_pass
self.high_pass = high_pass
self.t_r = t_r
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
self.noise_model = noise_model
self.percent_signal_change = percent_signal_change
if self.percent_signal_change:
self.standardize = False
def fit(self, imgs, design_matrices):
""" Fit the GLM
1. does a masker job: fMRI_data -> Y
2. fit an ols regression to (Y, X)
3. fit an AR(1) regression of require
This results in an internal (labels_, regression_results_) parameters
Parameters
----------
imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/building_blocks/manipulating_mr_images.html#niimg.
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
design_matrices: pandas DataFrame or list of pandas DataFrames,
fMRI design matrices
"""
# First, learn the mask
if not isinstance(self.mask, NiftiMasker):
self.masker_ = NiftiMasker(
mask_img=self.mask, smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize, low_pass=self.low_pass,
high_pass=self.high_pass, mask_strategy='epi',
t_r=self.t_r, memory=self.memory,
verbose=max(0, self.verbose - 1),
target_shape=self.target_shape,
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask)
for param_name in ['target_affine', 'target_shape',
'smoothing_fwhm', 'low_pass', 'high_pass',
't_r', 'memory', 'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
# make design_matrices a list of arrays
if isinstance(design_matrices, (_basestring, pd.DataFrame)):
design_matrices_ = [design_matrices]
else:
design_matrices_ = [X for X in design_matrices]
design_matrices = []
for design_matrix in design_matrices_:
if isinstance(design_matrix, _basestring):
loaded = pd.read_csv(design_matrix, index_col=0)
design_matrices.append(loaded.values)
elif isinstance(design_matrix, pd.DataFrame):
design_matrices.append(design_matrix.values)
else:
raise TypeError(
'Design matrix can only be a pandas DataFrames or a'
'string. A %s was provided' % type(design_matrix))
# make imgs a list of Nifti1Images
if isinstance(imgs, (Nifti1Image, _basestring)):
imgs = [imgs]
if len(imgs) != len(design_matrices):
raise ValueError(
'len(imgs) %d does not match len(design_matrices) %d'
% (len(imgs), len(design_matrices)))
# Loop on imgs and design matrices
self.labels_, self.results_ = [], []
self.masker_.fit(imgs)
for X, img in zip(design_matrices, imgs):
Y = self.masker_.transform(img)
if self.percent_signal_change:
Y, _ = percent_mean_scaling(Y)
labels_, results_ = session_glm(
Y, X, noise_model=self.noise_model, bins=100)
self.labels_.append(labels_)
self.results_.append(results_)
return self
def transform(self, con_vals, contrast_type=None, contrast_name='',
output_z=True, output_stat=False, output_effects=False,
output_variance=False):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
In multi-session case, outputs the fixed effects map.
Parameters
----------
con_vals : array or list of arrays of shape (n_col) or (n_dim, n_col)
where ``n_col`` is the number of columns of the design matrix,
numerical definition of the contrast (one array per run)
contrast_type : {'t', 'F'}, optional
type of the contrast
contrast_name : str, optional
name of the contrast
output_z : bool, optional
Return or not the corresponding z-stat image
output_stat : bool, optional
Return or not the base (t/F) stat image
output_effects : bool, optional
Return or not the corresponding effect image
output_variance : bool, optional
Return or not the corresponding variance image
Returns
-------
output_images : list of Nifti1Images
The desired output images
"""
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
if isinstance(con_vals, np.ndarray):
con_vals = [con_vals]
if len(con_vals) != len(self.results_):
raise ValueError(
'contrasts must be a sequence of %d session contrasts' %
len(self.results_))
contrast = None
for i, (labels_, results_, con_val) in enumerate(zip(
self.labels_, self.results_, con_vals)):
if np.all(con_val == 0):
warn('Contrast for session %d is null' % i)
contrast_ = compute_contrast(labels_, results_, con_val,
contrast_type)
if contrast is None:
contrast = contrast_
else:
contrast = contrast + contrast_
if output_z or output_stat:
# compute the contrast and stat
contrast.z_score()
# Prepare the returned images
do_outputs = [output_z, output_stat, output_effects, output_variance]
estimates = ['z_score_', 'stat_', 'effect', 'variance']
descrips = ['z statistic', 'Statistical value', 'Estimated effect',
'Estimated variance']
output_images = []
for do_output, estimate, descrip in zip(
do_outputs, estimates, descrips):
if not do_output:
continue
estimate_ = getattr(contrast, estimate)
if estimate_.ndim == 3:
shape_ = estimate_.shape
estimate_ = np.reshape(estimate_,
(shape_[0] * shape_[1], shape_[2]))
output = self.masker_.inverse_transform(estimate_)
output.get_header()['descrip'] = (
'%s of contrast %s' % (descrip, contrast_name))
output_images.append(output)
return output_images
def fit_transform(
self, design_matrices, fmri_images, con_vals, contrast_type=None,
contrast_name='', output_z=True, output_stat=False,
output_effects=False, output_variance=False):
""" Fit then transform. For more details,
see FirstLevelGLM.fit and FirstLevelGLM.transform documentation"""
return self.fit(design_matrices, fmri_images).transform(
con_vals, contrast_type, contrast_name, output_z=True,
output_stat=False, output_effects=False, output_variance=False)
labels = [0 if label==-1 else label for label in labels]
AT_labels = [0 if label==-1 else label for label in AT_labels]
labels = np.int32(labels)
AT_labels = np.int32(AT_labels)
# contrasts are IN ORDER -> shuffle!
new_inds = np.arange(0, X_task.shape[0])
np.random.shuffle(new_inds)
X_task = X_task[new_inds]
labels = labels[new_inds]
# subs = subs[new_inds]
# Resting state data
# Modified by @Gabighz
X_rest = nifti_masker.transform(rest_img)
# X_rest = nifti_masker.transform('dump_rs_spca_s12_tmp')
rs_spca_data = nib.load(rest_img) # was: joblib.load('dump_rs_spca_s12_tmp')
rs_spca_niis = nib.Nifti1Image(rs_spca_data,
nifti_masker.mask_img_.affine)
#X_rest = nifti_masker.transform(rs_spca_niis)
del rs_spca_niis
del rs_spca_data
X_task = StandardScaler().fit_transform(X_task)
X_rest = StandardScaler().fit_transform(X_rest)
# ARCHI Task
# NOTE: Used to test out-of-dataset performance
# was AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm')
AT_niis = nib.load(ARCHI_out_of_dataset)
import nibabel
pl.figure()
pl.axis('off')
pl.imshow(np.rot90(nibabel.load(dataset.func[0]).get_data()[..., 20, 0]),
interpolation='nearest',
cmap=pl.cm.gray)
ma = np.ma.masked_equal(mask, False)
pl.imshow(np.rot90(ma[..., 20]),
interpolation='nearest',
cmap=pl.cm.autumn,
alpha=0.5)
pl.title("Mask")
### Preprocess data ###########################################################
nifti_masker.fit(dataset.func[0])
fmri_masked = nifti_masker.transform(dataset.func[0])
### Run an algorithm ##########################################################
from sklearn.decomposition import FastICA
n_components = 20
ica = FastICA(n_components=n_components, random_state=42)
components_masked = ica.fit_transform(fmri_masked.T).T
### Reverse masking ###########################################################
components = nifti_masker.inverse_transform(components_masked)
### Show results ##############################################################
components_data = np.ma.masked_equal(components.get_data(), 0)
pl.figure()
pl.axis('off')
pl.imshow(np.rot90(nibabel.load(dataset.func[0]).get_data()[..., 20, 0]),
from glob import glob
import numpy as np
import os
from nilearn.image import new_img_like
from nilearn.input_data import NiftiMasker
from nilearn.image import resample_to_img
example_img = nib.load(os.path.join("movie_files", "sherlock_movie_s1.nii"))
mask_img = nib.load("sherlock_roi.nii.gz")
mask_img = resample_to_img(mask_img, example_img, interpolation="nearest")
masker = NiftiMasker(
mask_img=mask_img,
standardize=True,
smoothing_fwhm=None,
detrend=True,
high_pass=1.0 / 140,
t_r=1.5,
).fit()
# Let us first split sherlock data into 5 runs
for i, f in enumerate(glob(os.path.join("movie_files", "*"))):
niimg = nib.load(f)
X = niimg.get_data()
X = np.array_split(X, 5, axis=-1)
for j, x in enumerate(X):
os.makedirs("masked_movie_files", exist_ok=True)
np.save(
os.path.join("masked_movie_files", "sub%i_run%i" % (i, j)),
masker.transform(new_img_like(niimg, x)).T,
)
def fit(self, imgs, confounds=None):
"""
Compute the mask and the dynamic parcels across datasets.
Parameters
----------
imgs: list of Niimg-like objects
See http://nilearn.github.io/manipulating_images/input_output.html
Data on which the mask is calculated. If this is a list,
the affine is considered the same for all.
confounds: list of CSV file paths or 2D matrices
This parameter is passed to nilearn.signal.clean. Please see the
related documentation for details. Should match with the list
of imgs given.
Returns
-------
self: object
Returns the instance itself. Contains attributes listed
at the object level.
"""
self.masker_ = check_embedded_nifti_masker(self)
imgs, confounds = self._sanitize_imgs(imgs, confounds)
# Avoid warning with imgs != None
# if masker_ has been provided a mask_img
if self.masker_.mask_img is None:
self.masker_.fit(imgs)
else:
self.masker_.fit()
self.mask_img_ = self.masker_.mask_img_
# Load grey_matter segmentation
if self.grey_matter == "MNI":
mni = datasets.fetch_icbm152_2009()
self.grey_matter = mni.gm
if self.grey_matter is not None:
masker_anat = NiftiMasker(
mask_img=self.mask_img_, smoothing_fwhm=self.smoothing_fwhm
)
masker_anat.fit(self.grey_matter)
grey_matter = masker_anat.transform(self.grey_matter)
self.weights_grey_matter_ = (
1 - grey_matter
) + self.std_grey_matter * grey_matter
else:
self.weights_grey_matter_ = None
# Control random number generation
self.random_state = check_random_state(self.random_state)
# Check that number of batches is reasonable
if self.n_batch > len(imgs):
warnings.warn(
"{0} batches were requested, but only {1} datasets available. Using one dataset per batch instead.".format(
self.n_batch, len(imgs)
)
)
self.n_batch = len(imgs)
# mask_and_reduce step
if self.n_batch > 1:
stab_maps, dwell_time = self._mask_and_reduce_batch(imgs, confounds)
else:
stab_maps, dwell_time = self._mask_and_reduce(imgs, confounds)
# Return components
self.components_ = stab_maps
self.dwell_time_ = dwell_time
# Create embedding
self.embedding = Embedding(stab_maps.todense())
return self
masker.fit()
n_files = len(nii_paths)
n_vox = r_mask.get_data().sum()
if op.exists('dump_FS.npy'):
FS = np.load('dump_FS.npy')
cond_labels = np.load('dump_cond.npy')
sub_labels = np.load('dump_subs.npy')
else:
FS = np.zeros((n_files, n_vox))
for i_nii in range(n_files):
print('Loading nifti into memory: %i/%i' % (i_nii + 1, n_files))
data = np.nan_to_num(nib.load(nii_paths[i_nii]).get_data())
nii = nib.Nifti1Image(data, r_mask.get_affine())
cur_1d_data = masker.transform(nii)
FS[i_nii, :] = cur_1d_data
# save feature space to disk
np.save('dump_FS', FS)
np.save('dump_cond', cond_labels)
np.save('dump_subs', sub_labels)
from sklearn.preprocessing import StandardScaler
FS = StandardScaler().fit_transform(FS)
labels = cond_labels
# type conversion
FS = np.float32(FS)
labels = np.int32(labels)
def map_threshold(stat_img=None,
mask_img=None,
level=.001,
height_control='fpr',
cluster_threshold=0):
""" Compute the required threshold level and return the thresholded map
Parameters
----------
stat_img : Niimg-like object or None, optional
statistical image (presumably in z scale)
whenever height_control is 'fpr' or None,
stat_img=None is acceptable.
If it is 'fdr' or 'bonferroni', an error is raised if stat_img is None.
mask_img : Niimg-like object, optional,
mask image
level: float, optional
number controling the thresholding (either a p-value or z-scale value).
Not to be confused with the z-scale threshold: level can be a p-values,
e.g. "0.05" or another type of number depending on the
height_control parameter. The z-scale threshold is actually returned by
the function.
height_control: string, or None optional
false positive control meaning of cluster forming
threshold: 'fpr'|'fdr'|'bonferroni'\|None
cluster_threshold : float, optional
cluster size threshold. In the returned thresholded map,
sets of connected voxels (`clusters`) with size smaller
than this number will be removed.
Returns
-------
thresholded_map : Nifti1Image,
the stat_map thresholded at the prescribed voxel- and cluster-level
threshold: float,
the voxel-level threshold used actually
Note
----
If the input image is not z-scaled (i.e. some z-transformed statistic)
the computed threshold is not rigorous and likely meaningless
"""
# Check that height_control is correctly specified
if height_control not in ['fpr', 'fdr', 'bonferroni', None]:
raise ValueError(
"height control should be one of ['fpr', 'fdr', 'bonferroni', None]"
)
# if height_control is 'fpr' or None, we don't need to look at the data
# to compute the threhsold
if height_control == 'fpr':
threshold = norm.isf(level)
elif height_control is None:
threshold = level
# In this case, and is stat_img is None, we return
if stat_img is None:
if height_control in ['fpr', None]:
return None, threshold
else:
raise ValueError(
'Map_threshold requires stat_img not to be None'
'when the heigh_control procedure is bonferroni or fdr')
# Masking
if mask_img is None:
masker = NiftiMasker(mask_strategy='background').fit(stat_img)
else:
masker = NiftiMasker(mask_img=mask_img).fit()
stats = np.ravel(masker.transform(stat_img))
n_voxels = np.size(stats)
# Thresholding
if height_control == 'fdr':
threshold = fdr_threshold(stats, level)
elif height_control == 'bonferroni':
threshold = norm.isf(level / n_voxels)
stats *= (stats > threshold)
# embed it back to 3D grid
stat_map = masker.inverse_transform(stats).get_data()
# Extract connected components above threshold
label_map, n_labels = label(stat_map > threshold)
labels = label_map[masker.mask_img_.get_data() > 0]
for label_ in range(1, n_labels + 1):
if np.sum(labels == label_) < cluster_threshold:
stats[labels == label_] = 0
return masker.inverse_transform(stats), threshold
masker = NiftiMasker(
mask_img=mask_img, memory='nilearn_cache', memory_level=1)
masker = masker.fit()
# Images may fail to be transformed, and are of different shapes,
# so we need to transform one-by-one and keep track of failures.
X = []
is_usable = np.ones((len(images),), dtype=bool)
for index, image_path in enumerate(images):
# load image and remove nan and inf values.
# applying smooth_img to an image with fwhm=None simply cleans up
# non-finite values but otherwise doesn't modify the image.
image = smooth_img(image_path, fwhm=None)
try:
X.append(masker.transform(image))
except Exception as e:
meta = nv_data['images_meta'][index]
print("Failed to mask/reshape image: id: {0}; "
"name: '{1}'; collection: {2}; error: {3}".format(
meta.get('id'), meta.get('name'),
meta.get('collection_id'), e))
is_usable[index] = False
# Now reshape list into 2D matrix, and remove failed images from terms
X = np.vstack(X)
term_weights = term_weights[is_usable, :]
######################################################################
# Run ICA and map components to terms
print contr_name
mri_file = op.join(write_dir, 'mean_' + contr_name + '.nii.gz')
cur_imgs = stat_paths[key][y_contr[key] == i]
mean_nifti = mean_img(cur_imgs)
mean_nifti.to_filename(mri_file)
print "comparing %s with..." % mri_file
for i_compr, compr_name in enumerate(compr_names):
for i_ana in range(4):
fname_tag = compr_name + '*' + contr_name + '*.nii*'
search_str = op.join(reconstr_dir, 'compr2task_' +
analysis_modes[4 * i_key + i_ana] + '_' + PEN, fname_tag)
l = glob.glob(search_str)
print "- %s" % l
arr1 = masker.transform(mean_nifti)[0]
arr2 = masker.transform(l[0])[0]
rho, p = pearsonr(arr1, arr2)
mat_rho[key][i_compr, i, i_ana] = rho
L1 = np.sum(np.abs(arr1 - arr2))
mat_L1[key][i_compr, i, i_ana] = L1
print '-' * 80
# render mean
# my_min = 0.1
# my_max = 1.5
# brain = Brain("fsaverage", "split", "inflated",
# views=['lat', 'med'],
# config_opts=dict(background="black"))
def aCompCor_Muschelli(data_dir, out_dir, subj, task, ventricle_file):
'''
data_dir: [string] path to fmriprep output directory, containing subject directories.
out_dir: [string] path to directory to save aCompCor results to.
subj: [string] subject ID as it is exists in the fmriprep output directory (e.g. sub-001)
task: [string] task ID in the BIDS formatted file name.
e.g. 'stress' for file sub-001_task-stress_bold_space-MNI152NLin2009cAsym_preproc.nii.gz
If a task has multiple runs, include the run in this variable.
e.g. rest_run-01 for sub-001_task-rest_run-01_bold_space-MNI152NLin2009cAsym_preproc.nii.gz
ventricle_file: [string] path and name of ALVIN ventricle mask nifti file.
Download from https://sites.google.com/site/mrilateralventricle/
'''
import os
import numpy as np
import pandas as pd
import nibabel as nib
from scipy.ndimage.morphology import binary_erosion
from nilearn.image import resample_img, math_img
from nilearn.input_data import NiftiMasker
from sklearn.decomposition import PCA
print(
'Performing aCompCor (Muschelli et al. 2014, Neuroimage) on \n subject: %s \n task: %s'
% (subj, task))
if not os.path.isdir(os.path.join(out_dir, subj, 'PCA')):
os.makedirs(os.path.join(out_dir, subj, 'PCA'))
if not os.path.isdir(os.path.join(out_dir, subj, 'masks')):
os.makedirs(os.path.join(out_dir, subj, 'masks'))
##### White Matter
WM_file = nib.load(
os.path.join(
data_dir, subj, 'anat', subj +
'_T1w_space-MNI152NLin2009cAsym_class-WM_probtissue.nii.gz'))
WM_mask = WM_file.get_fdata()
WM_mask[WM_mask < (np.max(WM_mask) * .99)] = 0
WM_mask[WM_mask >= (np.max(WM_mask) * .99)] = 1
WM_mask = binary_erosion(WM_mask).astype(WM_mask.dtype)
nib.save(
nib.Nifti1Image(WM_mask, WM_file.affine, WM_file.header),
os.path.join(out_dir, subj, 'masks',
subj + '_task-' + task + '_WM-aCompCorMask.nii.gz'))
##### Cerebral Spinal Fluid
CSF_file = nib.load(
os.path.join(
data_dir, subj, 'anat', subj +
'_T1w_space-MNI152NLin2009cAsym_class-CSF_probtissue.nii.gz'))
CSF_mask = CSF_file.get_fdata()
CSF_mask[CSF_mask < (np.max(CSF_mask) * .99)] = 0
CSF_mask[CSF_mask >= (np.max(CSF_mask) * .99)] = 1
ventricle_mask = resample_img(nib.load(ventricle_file),
target_affine=CSF_file.affine,
target_shape=CSF_file.shape,
interpolation='nearest')
CSF_mask = np.multiply(CSF_mask, ventricle_mask.get_fdata())
nib.save(
nib.Nifti1Image(CSF_mask, CSF_file.affine, CSF_file.header),
os.path.join(out_dir, subj, 'masks',
subj + '_task-' + task + '_CSF-aCompCorMask.nii.gz'))
##### Mask functional data and perform PCA.
func_file = nib.load(
os.path.join(
data_dir, subj, 'func', subj + '_task-' + task +
'_bold_space-MNI152NLin2009cAsym_preproc.nii.gz'))
conf_file = os.path.join(data_dir, subj, 'func',
subj + '_task-' + task + '_bold_confounds.tsv')
CSF_fit = resample_img(nib.load(
os.path.join(out_dir, subj, 'masks',
subj + '_task-' + task + '_CSF-aCompCorMask.nii.gz')),
target_affine=func_file.affine,
target_shape=func_file.shape[0:3],
interpolation='nearest')
WM_fit = resample_img(nib.load(
os.path.join(out_dir, subj, 'masks',
subj + '_task-' + task + '_WM-aCompCorMask.nii.gz')),
target_affine=func_file.affine,
target_shape=func_file.shape[0:3],
interpolation='nearest')
# White Matter PCA
niftimask = NiftiMasker(dtype='float32')
niftimask.fit(math_img('img > 0', img=WM_fit))
func_WM = niftimask.transform(func_file)
runPCA = PCA()
PCA_WM = runPCA.fit(func_WM)
WM_out = pd.DataFrame(PCA_WM.components_)
WM_out.columns = [
'aCompCorWMnoFilter' + str(col) for col in WM_out.columns
]
WM_out.to_csv(os.path.join(out_dir, subj, 'PCA',
subj + '_task-' + task + '_WM-aCompCor.csv'),
index=False)
# CSF PCA
niftimask = NiftiMasker(dtype='float32')
niftimask.fit(math_img('img > 0', img=CSF_fit))
func_CSF = niftimask.transform(func_file)
runPCA = PCA()
PCA_CSF = runPCA.fit(func_CSF)
CSF_out = pd.DataFrame(PCA_CSF.components_)
CSF_out.columns = [
'aCompCorCSFnoFilter' + str(col) for col in CSF_out.columns
]
CSF_out.to_csv(os.path.join(out_dir, subj, 'PCA',
subj + '_task-' + task + '_CSF-aCompCor.csv'),
index=False)
# % Variance components
var_out = pd.DataFrame({
'var_WM':
PCA_WM.explained_variance_,
'percent_var_WM':
PCA_WM.explained_variance_ratio_,
'var_CSF':
PCA_CSF.explained_variance_,
'percent_var_CSF':
PCA_CSF.explained_variance_ratio_,
})
var_out.to_csv(
os.path.join(out_dir, subj, 'PCA',
subj + '_task-' + task + '_variance-aCompCor.csv'))
anat_filename = haxby_dataset.anat[subject_idx]
anat_img = nibabel.load(anat_filename)
fmri_filename = haxby_dataset.func[subject_idx]
fmri_raw_img = nibabel.load(fmri_filename)
shared_affine = fmri_raw_img.get_affine()
print("Build a mask based on the activations.")
epi_masker = NiftiMasker(mask_strategy='epi', detrend=True, standardize=True)
epi_masker = mem.cache(epi_masker.fit)(fmri_raw_img)
plot_roi(epi_masker.mask_img_,
title='EPI mask',
cut_coords=cut_coords)
print("Normalize the (transformed) data") # zscore per pixel, over examples.
from scipy import stats
fmri_masked_vectors = epi_masker.transform(fmri_raw_img)
import pdb; pdb.set_trace()
fmri_normed_vectors = mem.cache(stats.mstats.zscore)(fmri_masked_vectors, axis=0)
fmri_normed_img = epi_masker.inverse_transform(fmri_normed_vectors)
print("Smooth the (spatial) data.")
from nilearn import image
fmri_smooth_img = mem.cache(image.smooth_img)(fmri_normed_img, fwhm=1)
print("Mask the MRI data.")
masked_fmri_vectors = mem.cache(epi_masker.transform)(fmri_smooth_img)
fmri_masked_img = epi_masker.inverse_transform(masked_fmri_vectors)
print("Plot the mean image.")
fig_id = plt.subplot(2, 1, 1)
mean_img = mem.cache(image.mean_img)(fmri_masked_img)
class SecondLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for multiple subject
fMRI data
Parameters
----------
mask_img: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters. Automatic mask computation assumes first level imgs have
already been masked.
target_affine : 3x3 or 4x4 matrix, optional
This parameter is passed to :func:`nilearn.image.resample_img`.
Please see the related documentation for details.
target_shape : 3-tuple of integers, optional
This parameter is passed to :func:`nilearn.image.resample_img`.
Please see the related documentation for details.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints final computation time.
If 2 prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
"""
def __init__(self,
mask_img=None,
target_affine=None,
target_shape=None,
smoothing_fwhm=None,
memory=Memory(None),
memory_level=1,
verbose=0,
n_jobs=1,
minimize_memory=True):
self.mask_img = mask_img
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, str):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
self.second_level_input_ = None
self.confounds_ = None
def fit(self, second_level_input, confounds=None, design_matrix=None):
""" Fit the second-level GLM
1. create design matrix
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
second_level_input: list of `FirstLevelModel` objects or pandas
DataFrame or list of Niimg-like objects.
Giving FirstLevelModel objects will allow to easily compute
the second level contast of arbitrary first level contrasts thanks
to the first_level_contrast argument of the compute_contrast
method. Effect size images will be computed for each model to
contrast at the second level.
If a pandas DataFrame, then they have to contain subject_label,
map_name and effects_map_path. It can contain multiple maps that
would be selected during contrast estimation with the argument
first_level_contrast of the compute_contrast function. The
DataFrame will be sorted based on the subject_label column to avoid
order inconsistencies when extracting the maps. So the rows of the
automatically computed design matrix, if not provided, will
correspond to the sorted subject_label column.
If list of Niimg-like objects then this is taken literally as Y
for the model fit and design_matrix must be provided.
confounds: pandas DataFrame, optional
Must contain a subject_label column. All other columns are
considered as confounds and included in the model. If
design_matrix is provided then this argument is ignored.
The resulting second level design matrix uses the same column
names as in the given DataFrame for confounds. At least two columns
are expected, "subject_label" and at least one confound.
design_matrix: pandas DataFrame, optional
Design matrix to fit the GLM. The number of rows
in the design matrix must agree with the number of maps derived
from second_level_input.
Ensure that the order of maps given by a second_level_input
list of Niimgs matches the order of the rows in the design matrix.
"""
# Prevent circular imports between reporting & stats module
from nilearn.input_data import NiftiMasker # noqa
# check second_level_input
_check_second_level_input(second_level_input,
design_matrix,
confounds=confounds)
# check confounds
_check_confounds(confounds)
# check design matrix
_check_design_matrix(design_matrix)
# sort a pandas dataframe by subject_label to avoid inconsistencies
# with the design matrix row order when automatically extracting maps
if isinstance(second_level_input, pd.DataFrame):
columns = second_level_input.columns.tolist()
column_index = columns.index('subject_label')
sorted_matrix = sorted(second_level_input.values,
key=lambda x: x[column_index])
sorted_input = pd.DataFrame(sorted_matrix, columns=columns)
second_level_input = sorted_input
self.second_level_input_ = second_level_input
self.confounds_ = confounds
# Report progress
t0 = time.time()
if self.verbose > 0:
sys.stderr.write("Fitting second level model. "
"Take a deep breath\r")
# Select sample map for masker fit and get subjects_label for design
if isinstance(second_level_input, pd.DataFrame):
sample_map = second_level_input['effects_map_path'][0]
labels = second_level_input['subject_label']
subjects_label = labels.values.tolist()
elif isinstance(second_level_input, Nifti1Image):
sample_map = mean_img(second_level_input)
elif isinstance(second_level_input[0], FirstLevelModel):
sample_model = second_level_input[0]
sample_condition = sample_model.design_matrices_[0].columns[0]
sample_map = sample_model.compute_contrast(
sample_condition, output_type='effect_size')
labels = [model.subject_label for model in second_level_input]
subjects_label = labels
else:
# In this case design matrix had to be provided
sample_map = mean_img(second_level_input)
# Create and set design matrix, if not given
if design_matrix is None:
design_matrix = make_second_level_design_matrix(
subjects_label, confounds)
self.design_matrix_ = design_matrix
# Learn the mask. Assume the first level imgs have been masked.
if not isinstance(self.mask_img, NiftiMasker):
self.masker_ = NiftiMasker(mask_img=self.mask_img,
target_affine=self.target_affine,
target_shape=self.target_shape,
smoothing_fwhm=self.smoothing_fwhm,
memory=self.memory,
verbose=max(0, self.verbose - 1),
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask_img)
for param_name in ['smoothing_fwhm', 'memory', 'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(sample_map)
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of second level model done in "
"%i seconds\n" % (time.time() - t0))
return self
def compute_contrast(self,
second_level_contrast=None,
first_level_contrast=None,
second_level_stat_type=None,
output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
Parameters
----------
second_level_contrast: str or array of shape (n_col), optional
Where ``n_col`` is the number of columns of the design matrix. The
string can be a formula compatible with `pandas.DataFrame.eval`.
Basically one can use the name of the conditions as they appear in
the design matrix of the fitted model combined with operators +-
and combined with numbers with operators +-`*`/. The default (None)
is accepted if the design matrix has a single column, in which case
the only possible contrast array((1)) is applied; when the design
matrix has multiple columns, an error is raised.
first_level_contrast: str or array of shape (n_col) with respect to
FirstLevelModel, optional
In case a list of FirstLevelModel was provided as
second_level_input, we have to provide a contrast to apply to
the first level models to get the corresponding list of images
desired, that would be tested at the second level. In case a
pandas DataFrame was provided as second_level_input this is the
map name to extract from the pandas dataframe map_name column.
It has to be a 't' contrast.
second_level_stat_type: {'t', 'F'}, optional
Type of the second level contrast
output_type: str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size', 'effect_variance' or 'all'
Returns
-------
output_image: Nifti1Image
The desired output image(s). If ``output_type == 'all'``, then
the output is a dictionary of images, keyed by the type of image.
"""
if self.second_level_input_ is None:
raise ValueError('The model has not been fit yet')
# check first_level_contrast
_check_first_level_contrast(self.second_level_input_,
first_level_contrast)
# check contrast and obtain con_val
con_val = _get_con_val(second_level_contrast, self.design_matrix_)
# check output type
# 'all' is assumed to be the final entry;
# if adding more, place before 'all'
valid_types = [
'z_score', 'stat', 'p_value', 'effect_size', 'effect_variance',
'all'
]
_check_output_type(output_type, valid_types)
# Get effect_maps appropriate for chosen contrast
effect_maps = _infer_effect_maps(self.second_level_input_,
first_level_contrast)
# Check design matrix X and effect maps Y agree on number of rows
_check_effect_maps(effect_maps, self.design_matrix_)
# Fit an Ordinary Least Squares regression for parametric statistics
Y = self.masker_.transform(effect_maps)
if self.memory:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
labels, results = mem_glm(Y,
self.design_matrix_.values,
n_jobs=self.n_jobs,
noise_model='ols')
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.labels_ = labels
self.results_ = results
# We compute contrast object
if self.memory:
mem_contrast = self.memory.cache(compute_contrast)
else:
mem_contrast = compute_contrast
contrast = mem_contrast(self.labels_, self.results_, con_val,
second_level_stat_type)
output_types = \
valid_types[:-1] if output_type == 'all' else [output_type]
outputs = {}
for output_type_ in output_types:
# We get desired output from contrast object
estimate_ = getattr(contrast, output_type_)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_val)
output.header['descrip'] = ('%s of contrast %s' %
(output_type, contrast_name))
outputs[output_type_] = output
return outputs if output_type == 'all' else output
'MATH-STORY', 'STORY-MATH',
'T-AVG', 'F-H', 'H-F',
'MATCH-REL', 'REL-MATCH',
'BODY-AVG', 'FACE-AVG', 'PLACE-AVG', 'TOOL-AVG',
'2BK-0BK'
]
mean_supp = np.zeros((18, mask_nvox))
from nilearn.image import resample_img
for itask, task in enumerate(HCP_contrasts):
cur_nii = op.join(means_path, 'mean_%s.nii.gz' % (task))
print(cur_nii)
res_nii = resample_img(cur_nii,
target_affine=nifti_masker.mask_img_.get_affine(),
target_shape=nifti_masker.mask_img_.shape)
task_mean = nifti_masker.transform(res_nii)
mean_supp[itask, :] = task_mean
mean_supp_z = zscore(mean_supp, axis=1)
# get classification weights
lr_supp = np.load(op.join(LR_DIR, 'V0comps.npy'))
lr_supp_z = zscore(lr_supp, axis=1)
# get LR/AE weights
WRITE_DIR = 'nips3mm_recovery'
lambs = [0.25, 0.5, 0.75, 1]
import re
from scipy.stats import pearsonr
for n_comp in [5]:
corr_means_lr = np.zeros((len(lambs), 18))
corr_means_lr_ae = np.zeros((len(lambs), 18))
def non_parametric_inference(second_level_input,
confounds=None,
design_matrix=None,
second_level_contrast=None,
mask=None,
smoothing_fwhm=None,
model_intercept=True,
n_perm=10000,
two_sided_test=False,
random_state=None,
n_jobs=1,
verbose=0):
"""Generate p-values corresponding to the contrasts provided
based on permutation testing. This fuction reuses the 'permuted_ols'
function Nilearn.
Parameters
----------
second_level_input: pandas DataFrame or list of Niimg-like objects.
If a pandas DataFrame, then they have to contain subject_label,
map_name and effects_map_path. It can contain multiple maps that
would be selected during contrast estimation with the argument
first_level_contrast of the compute_contrast function. The
DataFrame will be sorted based on the subject_label column to avoid
order inconsistencies when extracting the maps. So the rows of the
automatically computed design matrix, if not provided, will
correspond to the sorted subject_label column.
If list of Niimg-like objects then this is taken literally as Y
for the model fit and design_matrix must be provided.
confounds: pandas DataFrame, optional
Must contain a subject_label column. All other columns are
considered as confounds and included in the model. If
design_matrix is provided then this argument is ignored.
The resulting second level design matrix uses the same column
names as in the given DataFrame for confounds. At least two columns
are expected, "subject_label" and at least one confound.
design_matrix: pandas DataFrame, optional
Design matrix to fit the GLM. The number of rows
in the design matrix must agree with the number of maps derived
from second_level_input.
Ensure that the order of maps given by a second_level_input
list of Niimgs matches the order of the rows in the design matrix.
second_level_contrast: str or array of shape (n_col), optional
Where ``n_col`` is the number of columns of the design matrix.
The default (None) is accepted if the design matrix has a single
column, in which case the only possible contrast array((1)) is
applied; when the design matrix has multiple columns, an error is
raised.
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters. Automatic mask computation assumes first level imgs have
already been masked.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
model_intercept : bool,
If True, a constant column is added to the confounding variates
unless the tested variate is already the intercept.
n_perm : int,
Number of permutations to perform.
Permutations are costly but the more are performed, the more precision
one gets in the p-values estimation.
two_sided_test : boolean,
If True, performs an unsigned t-test. Both positive and negative
effects are considered; the null hypothesis is that the effect is zero.
If False, only positive effects are considered as relevant. The null
hypothesis is that the effect is zero or negative.
random_state : int or None,
Seed for random number generator, to have the same permutations
in each computing units.
n_jobs : int,
Number of parallel workers.
If -1 is provided, all CPUs are used.
A negative number indicates that all the CPUs except (abs(n_jobs) - 1)
ones will be used.
verbose: int, optional
verbosity level (0 means no message).
Returns
-------
neg_log_corrected_pvals_img: Nifti1Image
The image which contains negative logarithm of the
corrected p-values
"""
# Prevent circular imports between reporting & stats module
from nilearn.input_data import NiftiMasker # noqa
_check_second_level_input(second_level_input,
design_matrix,
flm_object=False,
df_object=False)
_check_confounds(confounds)
_check_design_matrix(design_matrix)
# Report progress
t0 = time.time()
if verbose > 0:
sys.stderr.write("Fitting second level model...")
# Select sample map for masker fit and get subjects_label for design
sample_map = mean_img(second_level_input)
# Learn the mask. Assume the first level imgs have been masked.
if not isinstance(mask, NiftiMasker):
masker = NiftiMasker(mask_img=mask,
smoothing_fwhm=smoothing_fwhm,
memory=Memory(None),
verbose=max(0, verbose - 1),
memory_level=1)
else:
masker = clone(mask)
if smoothing_fwhm is not None:
if getattr(masker, 'smoothing_fwhm') is not None:
warn('Parameter smoothing_fwhm of the masker overriden')
setattr(masker, 'smoothing_fwhm', smoothing_fwhm)
masker.fit(sample_map)
# Report progress
if verbose > 0:
sys.stderr.write("\nComputation of second level model done in "
"%i seconds\n" % (time.time() - t0))
# Check and obtain the contrast
contrast = _get_contrast(second_level_contrast, design_matrix)
# Get effect_maps
effect_maps = _infer_effect_maps(second_level_input, None)
# Check design matrix and effect maps agree on number of rows
_check_effect_maps(effect_maps, design_matrix)
# Obtain tested_var
if contrast in design_matrix.columns.tolist():
tested_var = np.asarray(design_matrix[contrast])
# Mask data
target_vars = masker.transform(effect_maps)
# Perform massively univariate analysis with permuted OLS
neg_log_pvals_permuted_ols, _, _ = permuted_ols(
tested_var,
target_vars,
model_intercept=model_intercept,
n_perm=n_perm,
two_sided_test=two_sided_test,
random_state=random_state,
n_jobs=n_jobs,
verbose=max(0, verbose - 1))
neg_log_corrected_pvals_img = masker.inverse_transform(
np.ravel(neg_log_pvals_permuted_ols))
return neg_log_corrected_pvals_img
class Masker(object):
"""
Class that takes a binary mask.nii file and allows us to use it
within a volumizer in order to reduce the dimensionality of our data in
realtime.
If we have other ROI masks (e.g. wm, csf), we can use them detrend the data
by setting them as orthogonals.
"""
def __init__(self, mask_img, center=None, radius=8):
self.mask_img = mask_img
self.masker = NiftiMasker(mask_img=mask_img)
self.fit = False
# set the mask center
if center is None:
self.center = self.find_center_of_mass(self.masker)
else:
self.center = center
print("Center=", center)
print("COM calc=", self.find_center_of_mass(self.masker))
# the radius of the mask, used for determining what data to read.
self.radius = radius
self.orthogonals = []
self.use_orthogonal = False
self.ortho_fits = []
def reduce_volume(self, volume, method='mean'):
if not self.fit:
self.masker.fit(volume)
if method == 'mean':
reduced = npm(self.masker.transform(volume['image']))
return reduced
def find_center_of_mass(self, niftimasker):
"""
Find the center of mass of an image given a nifti masker object
in the z plane. We can use this information to only select dicoms
we need in a DicomFilter object.
"""
com = measurements.center_of_mass(
nibabel.load(niftimasker.mask_img).get_data())
affine = nibabel.load(niftimasker.mask_img).affine
offset = affine[0:3, 3]
tcom = np.dot(affine[0:3, 0:3], com) + offset
return tcom[2]
def add_orthogonal(self, mask_img):
# add another mask_img to our orthogonals with get_orthogonal
self.use_orthogonal = True
self.orthogonals.append(NiftiMasker(mask_img=mask_img))
self.ortho_fits.append(False)
def get_orthogonals(self, volume):
"""
Return a list of ROI averages for a volume given a set of
orthogonal masks
"""
for i, fit in enumerate(self.ortho_fits):
if not fit:
self.orthogonals[i].fit(volume)
return [npm(x.transform(volume['image'])) for x in self.orthogonals]
class FirstLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model
for single session fMRI data.
Parameters
----------
t_r : float
This parameter indicates repetition times of the experimental runs.
In seconds. It is necessary to correctly consider times in the design
matrix. This parameter is also passed to nilearn.signal.clean.
Please see the related documentation for details.
slice_time_ref : float, optional (default 0.)
This parameter indicates the time of the reference slice used in the
slice timing preprocessing step of the experimental runs. It is
expressed as a percentage of the t_r (time repetition), so it can have
values between 0. and 1.
hrf_model : {'spm', 'spm + derivative', 'spm + derivative + dispersion',
'glover', 'glover + derivative', 'glover + derivative + dispersion',
'fir', None}
String that specifies the hemodynamic response function.
Defaults to 'glover'.
drift_model : string, optional
This parameter specifies the desired drift model for the design
matrices. It can be 'polynomial', 'cosine' or None.
high_pass : float, optional
This parameter specifies the cut frequency of the high-pass filter in
Hz for the design matrices. Used only if drift_model is 'cosine'.
drift_order : int, optional
This parameter specifices the order of the drift model (in case it is
polynomial) for the design matrices.
fir_delays : array of shape(n_onsets) or list, optional
In case of FIR design, yields the array of delays used in the FIR
model, in scans.
min_onset : float, optional
This parameter specifies the minimal onset relative to the design
(in seconds). Events that start before (slice_time_ref * t_r +
min_onset) are not considered.
mask_img : Niimg-like, NiftiMasker object or False, optional
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a NiftiMasker with default
parameters. If False is given then the data will not be masked.
target_affine : 3x3 or 4x4 matrix, optional
This parameter is passed to nilearn.image.resample_img.
Please see the related documentation for details.
target_shape : 3-tuple of integers, optional
This parameter is passed to nilearn.image.resample_img.
Please see the related documentation for details.
smoothing_fwhm : float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of
the spatial smoothing to apply to the signal.
memory : string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done.
Creates instance of joblib.Memory.
memory_level : integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
standardize : boolean, optional
If standardize is True, the time-series are centered and normed:
their variance is put to 1 in the time dimension.
signal_scaling : False, int or (int, int), optional,
If not False, fMRI signals are
scaled to the mean value of scaling_axis given,
which can be 0, 1 or (0, 1).
0 refers to mean scaling each voxel with respect to time,
1 refers to mean scaling each time point with respect to all voxels &
(0, 1) refers to scaling with respect to voxels and time,
which is known as grand mean scaling.
Incompatible with standardize (standardize=False is enforced when
signal_scaling is not False).
noise_model : {'ar1', 'ols'}, optional
The temporal variance model. Defaults to 'ar1'
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints progress by computation of
each run. If 2 prints timing details of masker and GLM. If 3
prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
subject_label : string, optional
This id will be used to identify a `FirstLevelModel` when passed to
a `SecondLevelModel` object.
Attributes
----------
labels_ : array of shape (n_voxels,),
a map of values on voxels used to identify the corresponding model
results_ : dict,
with keys corresponding to the different labels values.
Values are SimpleRegressionResults corresponding to the voxels,
if minimize_memory is True,
RegressionResults if minimize_memory is False
"""
def __init__(self, t_r=None, slice_time_ref=0., hrf_model='glover',
drift_model='cosine', high_pass=.01, drift_order=1,
fir_delays=[0], min_onset=-24, mask_img=None,
target_affine=None, target_shape=None, smoothing_fwhm=None,
memory=Memory(None), memory_level=1, standardize=False,
signal_scaling=0, noise_model='ar1', verbose=0, n_jobs=1,
minimize_memory=True, subject_label=None):
# design matrix parameters
self.t_r = t_r
self.slice_time_ref = slice_time_ref
self.hrf_model = hrf_model
self.drift_model = drift_model
self.high_pass = high_pass
self.drift_order = drift_order
self.fir_delays = fir_delays
self.min_onset = min_onset
# glm parameters
self.mask_img = mask_img
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, str):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.standardize = standardize
if signal_scaling is False:
self.signal_scaling = signal_scaling
elif signal_scaling in [0, 1, (0, 1)]:
self.scaling_axis = signal_scaling
self.signal_scaling = True
self.standardize = False
else:
raise ValueError('signal_scaling must be "False", "0", "1"'
' or "(0, 1)"')
self.noise_model = noise_model
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
# attributes
self.labels_ = None
self.results_ = None
self.subject_label = subject_label
def fit(self, run_imgs, events=None, confounds=None,
design_matrices=None):
""" Fit the GLM
For each run:
1. create design matrix X
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
run_imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/manipulating_images/input_output.html#inputing-data-file-names-or-image-objects # noqa:E501
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
events: pandas Dataframe or string or list of pandas DataFrames or
strings
fMRI events used to build design matrices. One events object
expected per run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
confounds: pandas Dataframe or string or list of pandas DataFrames or
strings
Each column in a DataFrame corresponds to a confound variable
to be included in the regression model of the respective run_img.
The number of rows must match the number of volumes in the
respective run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
design_matrices: pandas DataFrame or list of pandas DataFrames,
Design matrices that will be used to fit the GLM. If given it
takes precedence over events and confounds.
"""
# Local import to prevent circular imports
from nilearn.input_data import NiftiMasker # noqa
# Check arguments
# Check imgs type
if events is not None:
_check_events_file_uses_tab_separators(events_files=events)
if not isinstance(run_imgs, (list, tuple)):
run_imgs = [run_imgs]
if design_matrices is None:
if events is None:
raise ValueError('events or design matrices must be provided')
if self.t_r is None:
raise ValueError('t_r not given to FirstLevelModel object'
' to compute design from events')
else:
design_matrices = _check_run_tables(run_imgs, design_matrices,
'design_matrices')
# Check that number of events and confound files match number of runs
# Also check that events and confound files can be loaded as DataFrame
if events is not None:
events = _check_run_tables(run_imgs, events, 'events')
if confounds is not None:
confounds = _check_run_tables(run_imgs, confounds, 'confounds')
# Learn the mask
if self.mask_img is False:
# We create a dummy mask to preserve functionality of api
ref_img = check_niimg(run_imgs[0])
self.mask_img = Nifti1Image(np.ones(ref_img.shape[:3]),
ref_img.affine)
if not isinstance(self.mask_img, NiftiMasker):
self.masker_ = NiftiMasker(mask_img=self.mask_img,
smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize,
mask_strategy='epi',
t_r=self.t_r,
memory=self.memory,
verbose=max(0, self.verbose - 2),
target_shape=self.target_shape,
memory_level=self.memory_level
)
self.masker_.fit(run_imgs[0])
else:
if self.mask_img.mask_img_ is None and self.masker_ is None:
self.masker_ = clone(self.mask_img)
for param_name in ['target_affine', 'target_shape',
'smoothing_fwhm', 't_r', 'memory',
'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker'
' overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(run_imgs[0])
else:
self.masker_ = self.mask_img
# For each run fit the model and keep only the regression results.
self.labels_, self.results_, self.design_matrices_ = [], [], []
n_runs = len(run_imgs)
t0 = time.time()
for run_idx, run_img in enumerate(run_imgs):
# Report progress
if self.verbose > 0:
percent = float(run_idx) / n_runs
percent = round(percent * 100, 2)
dt = time.time() - t0
# We use a max to avoid a division by zero
if run_idx == 0:
remaining = 'go take a coffee, a big one'
else:
remaining = (100. - percent) / max(0.01, percent) * dt
remaining = '%i seconds remaining' % remaining
sys.stderr.write(
"Computing run %d out of %d runs (%s)\n"
% (run_idx + 1, n_runs, remaining))
# Build the experimental design for the glm
run_img = check_niimg(run_img, ensure_ndim=4)
if design_matrices is None:
n_scans = get_data(run_img).shape[3]
if confounds is not None:
confounds_matrix = confounds[run_idx].values
if confounds_matrix.shape[0] != n_scans:
raise ValueError('Rows in confounds does not match'
'n_scans in run_img at index %d'
% (run_idx,))
confounds_names = confounds[run_idx].columns.tolist()
else:
confounds_matrix = None
confounds_names = None
start_time = self.slice_time_ref * self.t_r
end_time = (n_scans - 1 + self.slice_time_ref) * self.t_r
frame_times = np.linspace(start_time, end_time, n_scans)
design = make_first_level_design_matrix(frame_times,
events[run_idx],
self.hrf_model,
self.drift_model,
self.high_pass,
self.drift_order,
self.fir_delays,
confounds_matrix,
confounds_names,
self.min_onset
)
else:
design = design_matrices[run_idx]
self.design_matrices_.append(design)
# Mask and prepare data for GLM
if self.verbose > 1:
t_masking = time.time()
sys.stderr.write('Starting masker computation \r')
Y = self.masker_.transform(run_img)
del run_img # Delete unmasked image to save memory
if self.verbose > 1:
t_masking = time.time() - t_masking
sys.stderr.write('Masker took %d seconds \n'
% t_masking)
if self.signal_scaling:
Y, _ = mean_scaling(Y, self.scaling_axis)
if self.memory:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
# compute GLM
if self.verbose > 1:
t_glm = time.time()
sys.stderr.write('Performing GLM computation\r')
labels, results = mem_glm(Y, design.values,
noise_model=self.noise_model,
bins=100, n_jobs=self.n_jobs)
if self.verbose > 1:
t_glm = time.time() - t_glm
sys.stderr.write('GLM took %d seconds \n' % t_glm)
self.labels_.append(labels)
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.results_.append(results)
del Y
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of %d runs done in %i seconds\n\n"
% (n_runs, time.time() - t0))
return self
def compute_contrast(self, contrast_def, stat_type=None,
output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
In multi-session case, outputs the fixed effects map.
Parameters
----------
contrast_def : str or array of shape (n_col) or list of (string or
array of shape (n_col))
where ``n_col`` is the number of columns of the design matrix,
(one array per run). If only one array is provided when there
are several runs, it will be assumed that the same contrast is
desired for all runs. The string can be a formula compatible with
`pandas.DataFrame.eval`. Basically one can use the name of the
conditions as they appear in the design matrix of the fitted model
combined with operators +- and combined with numbers
with operators +-`*`/.
stat_type : {'t', 'F'}, optional
type of the contrast
output_type : str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size', 'effect_variance' or 'all'
Returns
-------
output : Nifti1Image or dict
The desired output image(s). If ``output_type == 'all'``, then
the output is a dictionary of images, keyed by the type of image.
"""
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
if isinstance(contrast_def, (np.ndarray, str)):
con_vals = [contrast_def]
elif isinstance(contrast_def, (list, tuple)):
con_vals = contrast_def
else:
raise ValueError('contrast_def must be an array or str or list of'
' (array or str)')
# Translate formulas to vectors
for cidx, (con, design_mat) in enumerate(zip(con_vals,
self.design_matrices_)
):
design_columns = design_mat.columns.tolist()
if isinstance(con, str):
con_vals[cidx] = expression_to_contrast_vector(
con, design_columns)
n_runs = len(self.labels_)
if len(con_vals) != n_runs:
warn('One contrast given, assuming it for all %d runs' % n_runs)
con_vals = con_vals * n_runs
valid_types = ['z_score', 'stat', 'p_value', 'effect_size',
'effect_variance']
valid_types.append('all') # ensuring 'all' is the final entry.
if output_type not in valid_types:
raise ValueError(
'output_type must be one of {}'.format(valid_types))
contrast = _compute_fixed_effect_contrast(self.labels_, self.results_,
con_vals, stat_type)
output_types = (valid_types[:-1]
if output_type == 'all' else [output_type])
outputs = {}
for output_type_ in output_types:
estimate_ = getattr(contrast, output_type_)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_vals)
output.header['descrip'] = (
'%s of contrast %s' % (output_type_, contrast_name))
outputs[output_type_] = output
return outputs if output_type == 'all' else output
def _get_voxelwise_model_attribute(self, attribute,
result_as_time_series):
"""Transform RegressionResults instances within a dictionary
(whose keys represent the autoregressive coefficient under the 'ar1'
noise model or only 0.0 under 'ols' noise_model and values are the
RegressionResults instances) into input nifti space.
Parameters
----------
attribute : str
an attribute of a RegressionResults instance.
possible values include: resid, norm_resid, predicted,
SSE, r_square, MSE.
result_as_time_series : bool
whether the RegressionResult attribute has a value
per timepoint of the input nifti image.
Returns
-------
output : list
a list of Nifti1Image(s)
"""
# check if valid attribute is being accessed.
all_attributes = dict(vars(RegressionResults)).keys()
possible_attributes = [prop
for prop in all_attributes
if '__' not in prop
]
if attribute not in possible_attributes:
msg = ("attribute must be one of: "
"{attr}".format(attr=possible_attributes)
)
raise ValueError(msg)
if self.minimize_memory:
raise ValueError(
'To access voxelwise attributes like '
'R-squared, residuals, and predictions, '
'the `FirstLevelModel`-object needs to store '
'there attributes. '
'To do so, set `minimize_memory` to `False` '
'when initializing the `FirstLevelModel`-object.')
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
output = []
for design_matrix, labels, results in zip(self.design_matrices_,
self.labels_,
self.results_
):
if result_as_time_series:
voxelwise_attribute = np.zeros((design_matrix.shape[0],
len(labels))
)
else:
voxelwise_attribute = np.zeros((1, len(labels)))
for label_ in results:
label_mask = labels == label_
voxelwise_attribute[:, label_mask] = getattr(results[label_],
attribute)
output.append(self.masker_.inverse_transform(voxelwise_attribute))
return output
@setattr_on_read
def residuals(self):
"""Transform voxelwise residuals to the same shape
as the input Nifti1Image(s)
Returns
-------
output : list
a list of Nifti1Image(s)
"""
return self._get_voxelwise_model_attribute('resid',
result_as_time_series=True)
@setattr_on_read
def predicted(self):
"""Transform voxelwise predicted values to the same shape
as the input Nifti1Image(s)
Returns
-------
output : list
a list of Nifti1Image(s)
"""
return self._get_voxelwise_model_attribute('predicted',
result_as_time_series=True)
@setattr_on_read
def r_square(self):
"""Transform voxelwise r-squared values to the same shape
as the input Nifti1Image(s)
Returns
-------
output : list
a list of Nifti1Image(s)
"""
return self._get_voxelwise_model_attribute('r_square',
result_as_time_series=False
)
class SecondLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for multiple subject
fMRI data
Parameters
----------
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters. Automatic mask computation assumes first level imgs have
already been masked.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints final computation time.
If 2 prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
"""
def __init__(self, mask=None, smoothing_fwhm=None,
memory=Memory(None), memory_level=1, verbose=0,
n_jobs=1, minimize_memory=True):
self.mask = mask
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, _basestring):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
self.second_level_input_ = None
self.confounds_ = None
def fit(self, second_level_input, confounds=None, design_matrix=None):
""" Fit the second-level GLM
1. create design matrix
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
second_level_input: list of `FirstLevelModel` objects or pandas
DataFrame or list of Niimg-like objects.
Giving FirstLevelModel objects will allow to easily compute
the second level contast of arbitrary first level contrasts thanks
to the first_level_contrast argument of the compute_contrast
method. Effect size images will be computed for each model to
contrast at the second level.
If a pandas DataFrame, then they have to contain subject_label,
map_name and effects_map_path. It can contain multiple maps that
would be selected during contrast estimation with the argument
first_level_contrast of the compute_contrast function. The
DataFrame will be sorted based on the subject_label column to avoid
order inconsistencies when extracting the maps. So the rows of the
automatically computed design matrix, if not provided, will
correspond to the sorted subject_label column.
If list of Niimg-like objects then this is taken literally as Y
for the model fit and design_matrix must be provided.
confounds: pandas DataFrame, optional
Must contain a subject_label column. All other columns are
considered as confounds and included in the model. If
design_matrix is provided then this argument is ignored.
The resulting second level design matrix uses the same column
names as in the given DataFrame for confounds. At least two columns
are expected, "subject_label" and at least one confound.
design_matrix: pandas DataFrame, optional
Design matrix to fit the GLM. The number of rows
in the design matrix must agree with the number of maps derived
from second_level_input.
Ensure that the order of maps given by a second_level_input
list of Niimgs matches the order of the rows in the design matrix.
"""
# Check parameters
# check first level input
if isinstance(second_level_input, list):
if len(second_level_input) < 2:
raise ValueError('A second level model requires a list with at'
'least two first level models or niimgs')
# Check FirstLevelModel objects case
if isinstance(second_level_input[0], FirstLevelModel):
models_input = enumerate(second_level_input)
for model_idx, first_level_model in models_input:
if (first_level_model.labels_ is None or
first_level_model.results_ is None):
raise ValueError(
'Model %s at index %i has not been fit yet'
'' % (first_level_model.subject_label, model_idx))
if not isinstance(first_level_model, FirstLevelModel):
raise ValueError(' object at idx %d is %s instead of'
' FirstLevelModel object' %
(model_idx, type(first_level_model)))
if confounds is not None:
if first_level_model.subject_label is None:
raise ValueError(
'In case confounds are provided, first level '
'objects need to provide the attribute '
'subject_label to match rows appropriately.'
'Model at idx %d does not provide it. '
'To set it, you can do '
'first_level_model.subject_label = "01"'
'' % (model_idx))
# Check niimgs case
elif isinstance(second_level_input[0], (str, Nifti1Image)):
if design_matrix is None:
raise ValueError('List of niimgs as second_level_input'
' require a design matrix to be provided')
for model_idx, niimg in enumerate(second_level_input):
if not isinstance(niimg, (str, Nifti1Image)):
raise ValueError(' object at idx %d is %s instead of'
' Niimg-like object' %
(model_idx, type(niimg)))
# Check pandas dataframe case
elif isinstance(second_level_input, pd.DataFrame):
for col in ['subject_label', 'map_name', 'effects_map_path']:
if col not in second_level_input.columns:
raise ValueError('second_level_input DataFrame must have'
' columns subject_label, map_name and'
' effects_map_path')
# Make sure subject_label contain strings
second_level_columns = second_level_input.columns.tolist()
labels_index = second_level_columns.index('subject_label')
labels_dtype = second_level_input.dtypes[labels_index]
if not isinstance(labels_dtype, np.object):
raise ValueError('subject_label column must be of dtype '
'object instead of dtype %s' % labels_dtype)
elif isinstance(second_level_input, (str, Nifti1Image)):
if design_matrix is None:
raise ValueError('List of niimgs as second_level_input'
' require a design matrix to be provided')
second_level_input = check_niimg(niimg=second_level_input,
ensure_ndim=4)
else:
raise ValueError('second_level_input must be a list of'
' `FirstLevelModel` objects, a pandas DataFrame'
' or a list Niimg-like objects. Instead %s '
'was provided' % type(second_level_input))
# check confounds
if confounds is not None:
if not isinstance(confounds, pd.DataFrame):
raise ValueError('confounds must be a pandas DataFrame')
if 'subject_label' not in confounds.columns:
raise ValueError('confounds DataFrame must contain column'
'"subject_label"')
if len(confounds.columns) < 2:
raise ValueError('confounds should contain at least 2 columns'
'one called "subject_label" and the other'
'with a given confound')
# Make sure subject_label contain strings
labels_index = confounds.columns.tolist().index('subject_label')
labels_dtype = confounds.dtypes[labels_index]
if not isinstance(labels_dtype, np.object):
raise ValueError('subject_label column must be of dtype '
'object instead of dtype %s' % labels_dtype)
# check design matrix
if design_matrix is not None:
if not isinstance(design_matrix, pd.DataFrame):
raise ValueError('design matrix must be a pandas DataFrame')
# sort a pandas dataframe by subject_label to avoid inconsistencies
# with the design matrix row order when automatically extracting maps
if isinstance(second_level_input, pd.DataFrame):
columns = second_level_input.columns.tolist()
column_index = columns.index('subject_label')
sorted_matrix = sorted(
second_level_input.values, key=lambda x: x[column_index])
sorted_input = pd.DataFrame(sorted_matrix, columns=columns)
second_level_input = sorted_input
self.second_level_input_ = second_level_input
self.confounds_ = confounds
# Report progress
t0 = time.time()
if self.verbose > 0:
sys.stderr.write("Fitting second level model. "
"Take a deep breath\r")
# Select sample map for masker fit and get subjects_label for design
if isinstance(second_level_input, pd.DataFrame):
sample_map = second_level_input['effects_map_path'][0]
labels = second_level_input['subject_label']
subjects_label = labels.values.tolist()
elif isinstance(second_level_input, Nifti1Image):
sample_map = mean_img(second_level_input)
elif isinstance(second_level_input[0], FirstLevelModel):
sample_model = second_level_input[0]
sample_condition = sample_model.design_matrices_[0].columns[0]
sample_map = sample_model.compute_contrast(
sample_condition, output_type='effect_size')
labels = [model.subject_label for model in second_level_input]
subjects_label = labels
else:
# In this case design matrix had to be provided
sample_map = mean_img(second_level_input)
# Create and set design matrix, if not given
if design_matrix is None:
design_matrix = make_second_level_design_matrix(subjects_label,
confounds)
self.design_matrix_ = design_matrix
# Learn the mask. Assume the first level imgs have been masked.
if not isinstance(self.mask, NiftiMasker):
self.masker_ = NiftiMasker(
mask_img=self.mask, smoothing_fwhm=self.smoothing_fwhm,
memory=self.memory, verbose=max(0, self.verbose - 1),
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask)
for param_name in ['smoothing_fwhm', 'memory', 'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(sample_map)
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of second level model done in "
"%i seconds\n" % (time.time() - t0))
return self
def compute_contrast(
self, second_level_contrast=None, first_level_contrast=None,
second_level_stat_type=None, output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
Parameters
----------
second_level_contrast: str or array of shape (n_col), optional
Where ``n_col`` is the number of columns of the design matrix,
The string can be a formula compatible with the linear constraint
of the Patsy library. Basically one can use the name of the
conditions as they appear in the design matrix of
the fitted model combined with operators /\*+- and numbers.
Please check the patsy documentation for formula examples:
http://patsy.readthedocs.io/en/latest/API-reference.html#patsy.DesignInfo.linear_constraint
The default (None) is accepted if the design matrix has a single
column, in which case the only possible contrast array([1]) is
applied; when the design matrix has multiple columns, an error is
raised.
first_level_contrast: str or array of shape (n_col) with respect to
FirstLevelModel, optional
In case a list of FirstLevelModel was provided as
second_level_input, we have to provide a contrast to apply to
the first level models to get the corresponding list of images
desired, that would be tested at the second level. In case a
pandas DataFrame was provided as second_level_input this is the
map name to extract from the pandas dataframe map_name column.
It has to be a 't' contrast.
second_level_stat_type: {'t', 'F'}, optional
Type of the second level contrast
output_type: str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size' or 'effect_variance'
Returns
-------
output_image: Nifti1Image
The desired output image
"""
if self.second_level_input_ is None:
raise ValueError('The model has not been fit yet')
# first_level_contrast check
if isinstance(self.second_level_input_[0], FirstLevelModel):
if first_level_contrast is None:
raise ValueError('If second_level_input was a list of '
'FirstLevelModel, then first_level_contrast '
'is mandatory. It corresponds to the '
'second_level_contrast argument of the '
'compute_contrast method of FirstLevelModel')
# check contrast definition
if second_level_contrast is None:
if self.design_matrix_.shape[1] == 1:
second_level_contrast = np.ones([1])
else:
raise ValueError('No second-level contrast is specified.')
if isinstance(second_level_contrast, np.ndarray):
con_val = second_level_contrast
if np.all(con_val == 0):
raise ValueError('Contrast is null')
else:
design_info = DesignInfo(self.design_matrix_.columns.tolist())
constraint = design_info.linear_constraint(second_level_contrast)
con_val = constraint.coefs
# check output type
if isinstance(output_type, _basestring):
if output_type not in ['z_score', 'stat', 'p_value', 'effect_size',
'effect_variance']:
raise ValueError(
'output_type must be one of "z_score", "stat"'
', "p_value", "effect_size" or "effect_variance"')
else:
raise ValueError('output_type must be one of "z_score", "stat",'
' "p_value", "effect_size" or "effect_variance"')
# Get effect_maps appropriate for chosen contrast
effect_maps = _infer_effect_maps(self.second_level_input_,
first_level_contrast)
# Check design matrix X and effect maps Y agree on number of rows
if len(effect_maps) != self.design_matrix_.shape[0]:
raise ValueError(
'design_matrix does not match the number of maps considered. '
'%i rows in design matrix do not match with %i maps' %
(self.design_matrix_.shape[0], len(effect_maps)))
# Fit an Ordinary Least Squares regression for parametric statistics
Y = self.masker_.transform(effect_maps)
if self.memory:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
labels, results = mem_glm(Y, self.design_matrix_.values,
n_jobs=self.n_jobs, noise_model='ols')
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.labels_ = labels
self.results_ = results
# We compute contrast object
if self.memory:
mem_contrast = self.memory.cache(compute_contrast)
else:
mem_contrast = compute_contrast
contrast = mem_contrast(self.labels_, self.results_, con_val,
second_level_stat_type)
# We get desired output from contrast object
estimate_ = getattr(contrast, output_type)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_val)
output.header['descrip'] = (
'%s of contrast %s' % (output_type, contrast_name))
return output
contrasts = ['face_trusty-gender', 'expression_intention-gender']
elif task in [
'hcp_emotion', 'hcp_gambling', 'hcp_language', 'hcp_relational',
'hcp_social', 'hcp_motor', 'hcp_wm'
]:
contrasts = [c for c in contrasts if c in df_hcp['IBC name'].unique()]
image_dir = cache
if task in [
'ArchiStandard', 'ArchiSpatial', 'ArchiSocial', 'ArchiEmotional'
]:
image_dir = os.path.join(cache, 'archi')
for contrast in contrasts:
hcp_img = os.path.join(image_dir, 'mean_z_%s.nii.gz' % contrast)
imgs = task_df[task_df.contrast == contrast].path.values
x = masker.transform(imgs).mean(0)
X.append(x)
y = masker.transform(hcp_img)[0]
Y.append(y)
names.append(contrast)
pretty_names = [
CONTRASTS[CONTRASTS.contrast == name]['positive label'].values[0] +
' vs. ' + CONTRASTS[CONTRASTS.contrast == name]['negative label'].values[0]
for name in names
]
n_contrasts = len(X)
C = np.corrcoef(np.array(X), np.array(Y))
plt.figure(figsize=(16, 13.2))
ax = plt.subplot(111)
for axis in ['top', 'bottom', 'left', 'right']:
def compute_dices():
output_dir = expanduser('output/ibc/language44')
mask = join(output_dir, 'mask_ibc.nii.gz')
masker = NiftiMasker(mask_img=mask).fit()
imgs = sorted(glob.glob(join(output_dir, '*_thr.nii.gz')))
regex = re.compile(r'(?P<contrast>.*)_z_score_(?P<reduction>.*)_'
r'(?P<size>.*)_(?P<split>[1-9]*)_thr.nii.gz')
df = []
for img in imgs:
_, filename = os.path.split(img)
match = regex.match(filename)
if match:
contrast = match['contrast']
reduction = match['reduction']
size = int(match['size']) if match['size'] != 'none' else 'none'
split = int(match['split'])
df.append(
dict(contrast=contrast,
reduction=reduction,
size=size,
split=split,
filename=img))
df = pd.DataFrame(df)
intra_dices = []
ref_df = df.loc[df['reduction'] == 'none']
for contrast, sub_df in ref_df.groupby(by='contrast'):
n = sub_df.shape[0]
components = masker.transform(sub_df['filename']) != 0
dices = np.array([
dice_index(components[i], components[j]) for i in range(n - 1)
for j in range(i + 1, n)
])
print(contrast, dices)
intra_dices.append(
dict(contrast=contrast,
mean_dice=np.mean(dices),
std_dice=np.std(dices)))
intra_dices = pd.DataFrame(intra_dices)
stop
intra_dices.to_pickle(join(output_dir, 'intra_dices.pkl'))
inter_dices = []
for (contrast, reduction,
size), sub_df in df.groupby(by=['contrast', 'reduction', 'size']):
sub_ref_df = ref_df.loc[ref_df['contrast'] == contrast]
n = sub_df.shape[0]
components = masker.transform(sub_df['filename']) != 0
ref_components = masker.transform(sub_ref_df['filename']) != 0
dices = np.array(
[dice_index(components[i], ref_components[i]) for i in range(n)])
print(contrast, reduction, size, dices)
inter_dices.append(
dict(contrast=contrast,
reduction=reduction,
size=size,
mean_dice=np.mean(dices),
std_dice=np.std(dices)))
inter_dices = pd.DataFrame(inter_dices)
inter_dices.to_pickle(join(output_dir, 'inter_dices.pkl'))
def nii2cmu(nifti_file,
mask_file=None,
smooth=None,
zscore=False,
zscore_by_rest=False,
rest_starts=None,
rest_ends=None):
if zscore_by_rest:
rest_starts = rest_starts.strip('[]')
rest_starts = [int(s) for s in rest_starts.split(',')]
rest_ends = rest_ends.strip('[]')
rest_ends = [int(s) for s in rest_ends.split(',')]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
image = nib.load(nifti_file)
mask = NiftiMasker(mask_strategy='background',
smoothing_fwhm=smooth,
standardize=False)
if mask_file is None:
mask.fit(nifti_file)
else:
mask.fit(mask_file)
header = image.header
sform = image.get_sform()
voxel_size = header.get_zooms()
voxel_activations = np.float64(mask.transform(nifti_file)).transpose()
rest_activations = voxel_activations[:, rest_starts[0]:rest_ends[0]]
for i in range(1, len(rest_starts)):
rest_activations = np.hstack(
(rest_activations,
voxel_activations[:, rest_starts[i]:rest_ends[i]]))
standard_transform = sklearn.preprocessing.StandardScaler().fit(
rest_activations.T)
voxel_activations = standard_transform.transform(voxel_activations.T).T
voxel_coordinates = np.array(np.nonzero(
mask.mask_img_.dataobj)).transpose()
voxel_coordinates = np.hstack(
(voxel_coordinates, np.ones((voxel_coordinates.shape[0], 1))))
voxel_locations = (voxel_coordinates @ sform.T)[:, :3]
else:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
image = nib.load(nifti_file)
mask = NiftiMasker(mask_strategy='background',
smoothing_fwhm=smooth,
standardize=zscore)
if mask_file is None:
mask.fit(nifti_file)
else:
mask.fit(mask_file)
header = image.header
sform = image.get_sform()
voxel_size = header.get_zooms()
voxel_activations = np.float64(mask.transform(nifti_file)).transpose()
voxel_coordinates = np.array(np.nonzero(
mask.mask_img_.dataobj)).transpose()
voxel_coordinates = np.hstack(
(voxel_coordinates, np.ones((voxel_coordinates.shape[0], 1))))
voxel_locations = (voxel_coordinates @ sform.T)[:, :3]
return {'data': voxel_activations, 'R': voxel_locations}
class FirstLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for single session fMRI data
Parameters
----------
t_r : float
This parameter indicates repetition times of the experimental runs.
In seconds. It is necessary to correctly consider times in the design
matrix. This parameter is also passed to nilearn.signal.clean.
Please see the related documentation for details.
slice_time_ref : float, optional (default 0.)
This parameter indicates the time of the reference slice used in the
slice timing preprocessing step of the experimental runs. It is
expressed as a percentage of the t_r (time repetition), so it can have
values between 0. and 1.
hrf_model : string, optional
This parameter specifies the hemodynamic response function (HRF) for
the design matrices. It can be 'canonical', 'canonical with derivative'
or 'fir'.
drift_model : string, optional
This parameter specifies the desired drift model for the design
matrices. It can be 'polynomial', 'cosine' or None.
period_cut : float, optional
This parameter specifies the cut period of the low-pass filter in
seconds for the design matrices.
drift_order : int, optional
This parameter specifices the order of the drift model (in case it is
polynomial) for the design matrices.
fir_delays : array of shape(n_onsets) or list, optional
In case of FIR design, yields the array of delays used in the FIR
model, in seconds.
min_onset : float, optional
This parameter specifies the minimal onset relative to the design
(in seconds). Events that start before (slice_time_ref * t_r +
min_onset) are not considered.
mask : Niimg-like, NiftiMasker object or False, optional
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a NiftiMasker with default
parameters. If False is given then the data will not be masked.
target_affine : 3x3 or 4x4 matrix, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
target_shape : 3-tuple of integers, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
smoothing_fwhm : float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory : string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level : integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
standardize : boolean, optional
If standardize is True, the time-series are centered and normed:
their variance is put to 1 in the time dimension.
signal_scaling : False, int or (int, int), optional,
If not False, fMRI signals are scaled to the mean value of scaling_axis
given, which can be 0, 1 or (0, 1). 0 refers to mean scaling each voxel
with respect to time, 1 refers to mean scaling each time point with
respect to all voxels and (0, 1) refers to scaling with respect to
voxels and time, which is known as grand mean scaling.
Incompatible with standardize (standardize=False is enforced when
signal_scaling is not False).
noise_model : {'ar1', 'ols'}, optional
The temporal variance model. Defaults to 'ar1'
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints progress by computation of
each run. If 2 prints timing details of masker and GLM. If 3
prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
subject_label : string, optional
This id will be used to identify a `FirstLevelModel` when passed to
a `SecondLevelModel` object.
Attributes
----------
labels : array of shape (n_voxels,),
a map of values on voxels used to identify the corresponding model
results : dict,
with keys corresponding to the different labels values
values are RegressionResults instances corresponding to the voxels
"""
def __init__(self,
t_r=None,
slice_time_ref=0.,
hrf_model='glover',
drift_model='cosine',
period_cut=128,
drift_order=1,
fir_delays=[0],
min_onset=-24,
mask=None,
target_affine=None,
target_shape=None,
smoothing_fwhm=None,
memory=Memory(None),
memory_level=1,
standardize=False,
signal_scaling=0,
noise_model='ar1',
verbose=0,
n_jobs=1,
minimize_memory=True,
subject_label=None):
# design matrix parameters
self.t_r = t_r
self.slice_time_ref = slice_time_ref
self.hrf_model = hrf_model
self.drift_model = drift_model
self.period_cut = period_cut
self.drift_order = drift_order
self.fir_delays = fir_delays
self.min_onset = min_onset
# glm parameters
self.mask = mask
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, _basestring):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.standardize = standardize
if signal_scaling is False:
self.signal_scaling = signal_scaling
elif (type(signal_scaling) is not bool) and (signal_scaling
in [0, 1, (0, 1)]):
self.scaling_axis = signal_scaling
self.signal_scaling = True
self.standardize = False
else:
raise ValueError('signal_scaling must be "False", "0", "1"'
' or "(0, 1)"')
self.noise_model = noise_model
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
# attributes
self.labels_ = None
self.results_ = None
self.subject_label = subject_label
def fit(self, run_imgs, events=None, confounds=None, design_matrices=None):
""" Fit the GLM
For each run:
1. create design matrix X
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
run_imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/building_blocks/manipulating_mr_images.html#niimg.
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
events: pandas Dataframe or string or list of pandas DataFrames or
strings,
fMRI events used to build design matrices. One events object
expected per run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
confounds: pandas Dataframe or string or list of pandas DataFrames or
strings,
Each column in a DataFrame corresponds to a confound variable
to be included in the regression model of the respective run_img.
The number of rows must match the number of volumes in the
respective run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
design_matrices: pandas DataFrame or list of pandas DataFrames,
Design matrices that will be used to fit the GLM. If given it
takes precedence over events and confounds.
"""
# Check arguments
# Check imgs type
if not isinstance(run_imgs, (list, tuple)):
run_imgs = [run_imgs]
if design_matrices is None:
if events is None:
raise ValueError('events or design matrices must be provided')
if self.t_r is None:
raise ValueError('t_r not given to FirstLevelModel object'
' to compute design from events')
else:
design_matrices = _check_run_tables(run_imgs, design_matrices,
'design_matrices')
# Check that number of events and confound files match number of runs
# Also check that events and confound files can be loaded as DataFrame
if events is not None:
events = _check_run_tables(run_imgs, events, 'events')
if confounds is not None:
confounds = _check_run_tables(run_imgs, confounds, 'confounds')
# Learn the mask
if self.mask is False:
# We create a dummy mask to preserve functionality of api
ref_img = check_niimg(run_imgs[0])
self.mask = Nifti1Image(np.ones(ref_img.shape[:3]), ref_img.affine)
if not isinstance(self.mask, NiftiMasker):
self.masker_ = NiftiMasker(mask_img=self.mask,
smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize,
mask_strategy='epi',
t_r=self.t_r,
memory=self.memory,
verbose=max(0, self.verbose - 2),
target_shape=self.target_shape,
memory_level=self.memory_level)
self.masker_.fit(run_imgs[0])
else:
if self.mask.mask_img_ is None and self.masker_ is None:
self.masker_ = clone(self.mask)
for param_name in [
'target_affine', 'target_shape', 'smoothing_fwhm',
't_r', 'memory', 'memory_level'
]:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker'
' overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(run_imgs[0])
else:
self.masker_ = self.mask
# For each run fit the model and keep only the regression results.
self.labels_, self.results_, self.design_matrices_ = [], [], []
n_runs = len(run_imgs)
t0 = time.time()
for run_idx, run_img in enumerate(run_imgs):
# Report progress
if self.verbose > 0:
percent = float(run_idx) / n_runs
percent = round(percent * 100, 2)
dt = time.time() - t0
# We use a max to avoid a division by zero
if run_idx == 0:
remaining = 'go take a coffee, a big one'
else:
remaining = (100. - percent) / max(0.01, percent) * dt
remaining = '%i seconds remaining' % remaining
sys.stderr.write("Computing run %d out of %d runs (%s)\n" %
(run_idx + 1, n_runs, remaining))
# Build the experimental design for the glm
run_img = check_niimg(run_img, ensure_ndim=4)
if design_matrices is None:
n_scans = run_img.get_data().shape[3]
if confounds is not None:
confounds_matrix = confounds[run_idx].values
if confounds_matrix.shape[0] != n_scans:
raise ValueError('Rows in confounds does not match'
'n_scans in run_img at index %d' %
(run_idx, ))
confounds_names = confounds[run_idx].columns.tolist()
else:
confounds_matrix = None
confounds_names = None
start_time = self.slice_time_ref * self.t_r
end_time = (n_scans - 1 + self.slice_time_ref) * self.t_r
frame_times = np.linspace(start_time, end_time, n_scans)
design = make_design_matrix(frame_times, events[run_idx],
self.hrf_model, self.drift_model,
self.period_cut, self.drift_order,
self.fir_delays, confounds_matrix,
confounds_names, self.min_onset)
else:
design = design_matrices[run_idx]
self.design_matrices_.append(design)
# Mask and prepare data for GLM
if self.verbose > 1:
t_masking = time.time()
sys.stderr.write('Starting masker computation \r')
Y = self.masker_.transform(run_img)
if self.verbose > 1:
t_masking = time.time() - t_masking
sys.stderr.write('Masker took %d seconds \n' %
t_masking)
if self.signal_scaling:
Y, _ = mean_scaling(Y, self.scaling_axis)
if self.memory is not None:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
# compute GLM
if self.verbose > 1:
t_glm = time.time()
sys.stderr.write('Performing GLM computation\r')
labels, results = mem_glm(Y,
design.as_matrix(),
noise_model=self.noise_model,
bins=100,
n_jobs=self.n_jobs)
if self.verbose > 1:
t_glm = time.time() - t_glm
sys.stderr.write('GLM took %d seconds \n' % t_glm)
self.labels_.append(labels)
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.results_.append(results)
del Y
# Report progress
if self.verbose > 0:
sys.stderr.write(
"\nComputation of %d runs done in %i seconds\n\n" %
(n_runs, time.time() - t0))
return self
def compute_contrast(self,
contrast_def,
stat_type=None,
output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
In multi-session case, outputs the fixed effects map.
Parameters
----------
contrast_def : str or array of shape (n_col) or list of (string or
array of shape (n_col))
where ``n_col`` is the number of columns of the design matrix,
(one array per run). If only one array is provided when there
are several runs, it will be assumed that the same contrast is
desired for all runs. The string can be a formula compatible with
the linear constraint of the Patsy library. Basically one can use
the name of the conditions as they appear in the design matrix of
the fitted model combined with operators /*+- and numbers.
Please checks the patsy documentation for formula examples:
http://patsy.readthedocs.io/en/latest/API-reference.html#patsy.DesignInfo.linear_constraint
stat_type : {'t', 'F'}, optional
type of the contrast
output_type : str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size' or 'effect_variance'
Returns
-------
output : Nifti1Image
The desired output image
"""
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
if isinstance(contrast_def, (np.ndarray, str)):
con_vals = [contrast_def]
elif isinstance(contrast_def, (list, tuple)):
con_vals = contrast_def
else:
raise ValueError('contrast_def must be an array or str or list of'
' (array or str)')
# Translate formulas to vectors with patsy
design_info = DesignInfo(self.design_matrices_[0].columns.tolist())
for cidx, con in enumerate(con_vals):
if not isinstance(con, np.ndarray):
con_vals[cidx] = design_info.linear_constraint(con).coefs
n_runs = len(self.labels_)
if len(con_vals) != n_runs:
warn('One contrast given, assuming it for all %d runs' % n_runs)
con_vals = con_vals * n_runs
if isinstance(output_type, _basestring):
if output_type not in [
'z_score', 'stat', 'p_value', 'effect_size',
'effect_variance'
]:
raise ValueError(
'output_type must be one of "z_score", "stat",'
' "p_value","effect_size" or "effect_variance"')
else:
raise ValueError('output_type must be one of "z_score", "stat",'
' "p_value","effect_size" or "effect_variance"')
contrast = _fixed_effect_contrast(self.labels_, self.results_,
con_vals, stat_type)
estimate_ = getattr(contrast, output_type)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_vals)
output.header['descrip'] = ('%s of contrast %s' %
(output_type, contrast_name))
return output
def load_data_PercIm_whb(pipeline, subj, newmask):
# datapath=input("Please enter the path to the preprocessed data:")
# mask_path=input("Please enter the path to the MNI based mask files:")
# protocol_path=input("Please enter the path to the protocol files:")
# ExpType=input("Please enter experiment type you wish to analyze - Perc or Im:")
filename_pattern = ".nii.gz"
datapath = "/home/elena/ATTEND/validataset/data/PI"
mask_path = "/home/elena/ATTEND/MASKS"
print(subj)
print("Loading the data")
# print mask
#MNI SPACE
subj_path = os.path.join(datapath, subj)
#NATIVE SPACE
#subj_path=os.path.join(datapath, subjName, 'preprocessed_native')
print(subj_path)
# path, dirs, files = os.walk(subj_path).next()
n_run = 4 #int(len(dirs)) #int(len(files))
#NATIVE SPACE
#n_run = len(files)/2
#MNI SPACE
#n_run = int(len(dirs))#len(files)-2
print(n_run)
#because apart from run files, coregistration files are also saved in the preprocessed folder, might be subject to change
nifti_masker = NiftiMasker(
mask_img=newmask, detrend=True, standardize=True
) #, memory_level=3, memory="/home/elena/ATTEND/validataset/TEMP/")
#print current_mask.shape
masked_data = [0] * n_run
masked_data_scaled = [0] * n_run
min_max_scaler = preprocessing.MinMaxScaler()
for r in range(0, n_run):
#NATIVE SPACE
#run_image = nibabel.load(os.path.join(subj_path, subj+'_native_run'+str(r+1)+filename_pattern))
#smoothed
#run_image = nibabel.load(os.path.join(subj_path, subj+'_native_run'+str(r+1)+'_sps6'+filename_pattern))
if pipeline == 'fts':
run_image = nibabel.load(
os.path.join(
subj_path, 'r0' + str(r + 1), 'to_standard_fts', subj +
'.PI.r0' + str(r + 1) + '.tstost.fts' + filename_pattern))
# else:
#nibabel.load(os.path.join(subj_path, 'r0'+str(r+1), 'to_standard_fts', subj+'.PI''.r0'+str(r+1)+'.tstost.fts' +filename_pattern))
else:
if pipeline == 'stf':
run_image = nibabel.load(
os.path.join(
subj_path, 'r0' + str(r + 1), 'to_standard_stf',
'PI.' + subj + '.r0' + str(r + 1) + '.tost_stf' +
filename_pattern))
nifti_masker.fit(run_image)
masked_data[r] = nifti_masker.transform(run_image)
masked_data_scaled[r] = min_max_scaler.fit_transform(masked_data[r])
print(len(masked_data_scaled[r]))
print(masked_data_scaled[r].shape)
return n_run, masked_data_scaled, nifti_masker
def non_parametric_inference(
second_level_input, confounds=None, design_matrix=None,
second_level_contrast=None, mask=None, smoothing_fwhm=None,
model_intercept=True, n_perm=10000, two_sided_test=False,
random_state=None, n_jobs=1, verbose=0):
"""Generate p-values corresponding to the contrasts provided
based on permutation testing. This fuction reuses the 'permuted_ols'
function Nilearn.
Parameters
----------
second_level_input: pandas DataFrame or list of Niimg-like objects.
If a pandas DataFrame, then they have to contain subject_label,
map_name and effects_map_path. It can contain multiple maps that
would be selected during contrast estimation with the argument
first_level_contrast of the compute_contrast function. The
DataFrame will be sorted based on the subject_label column to avoid
order inconsistencies when extracting the maps. So the rows of the
automatically computed design matrix, if not provided, will
correspond to the sorted subject_label column.
If list of Niimg-like objects then this is taken literally as Y
for the model fit and design_matrix must be provided.
confounds: pandas DataFrame, optional
Must contain a subject_label column. All other columns are
considered as confounds and included in the model. If
design_matrix is provided then this argument is ignored.
The resulting second level design matrix uses the same column
names as in the given DataFrame for confounds. At least two columns
are expected, "subject_label" and at least one confound.
design_matrix: pandas DataFrame, optional
Design matrix to fit the GLM. The number of rows
in the design matrix must agree with the number of maps derived
from second_level_input.
Ensure that the order of maps given by a second_level_input
list of Niimgs matches the order of the rows in the design matrix.
second_level_contrast: str or array of shape (n_col), optional
Where ``n_col`` is the number of columns of the design matrix.
The default (None) is accepted if the design matrix has a single
column, in which case the only possible contrast array([1]) is
applied; when the design matrix has multiple columns, an error is
raised.
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters. Automatic mask computation assumes first level imgs have
already been masked.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
model_intercept : bool,
If True, a constant column is added to the confounding variates
unless the tested variate is already the intercept.
n_perm : int,
Number of permutations to perform.
Permutations are costly but the more are performed, the more precision
one gets in the p-values estimation.
two_sided_test : boolean,
If True, performs an unsigned t-test. Both positive and negative
effects are considered; the null hypothesis is that the effect is zero.
If False, only positive effects are considered as relevant. The null
hypothesis is that the effect is zero or negative.
random_state : int or None,
Seed for random number generator, to have the same permutations
in each computing units.
n_jobs : int,
Number of parallel workers.
If -1 is provided, all CPUs are used.
A negative number indicates that all the CPUs except (abs(n_jobs) - 1)
ones will be used.
verbose: int, optional
verbosity level (0 means no message).
Returns
-------
neg_log_corrected_pvals_img: Nifti1Image
The image which contains negative logarithm of the
corrected p-values
"""
_check_second_level_input(second_level_input, design_matrix,
flm_object=False, df_object=False)
_check_confounds(confounds)
_check_design_matrix(design_matrix)
# Report progress
t0 = time.time()
if verbose > 0:
sys.stderr.write("Fitting second level model...")
# Select sample map for masker fit and get subjects_label for design
sample_map = mean_img(second_level_input)
# Learn the mask. Assume the first level imgs have been masked.
if not isinstance(mask, NiftiMasker):
masker = NiftiMasker(
mask_img=mask, smoothing_fwhm=smoothing_fwhm,
memory=Memory(None), verbose=max(0, verbose - 1),
memory_level=1)
else:
masker = clone(mask)
if smoothing_fwhm is not None:
if getattr(masker, 'smoothing_fwhm') is not None:
warn('Parameter smoothing_fwhm of the masker overriden')
setattr(masker, 'smoothing_fwhm', smoothing_fwhm)
masker.fit(sample_map)
# Report progress
if verbose > 0:
sys.stderr.write("\nComputation of second level model done in "
"%i seconds\n" % (time.time() - t0))
# Check and obtain the contrast
contrast = _get_contrast(second_level_contrast, design_matrix)
# Get effect_maps
effect_maps = _infer_effect_maps(second_level_input, None)
# Check design matrix and effect maps agree on number of rows
_check_effect_maps(effect_maps, design_matrix)
# Obtain tested_var
if contrast in design_matrix.columns.tolist():
tested_var = np.asarray(design_matrix[contrast])
# Mask data
target_vars = masker.transform(effect_maps)
# Perform massively univariate analysis with permuted OLS
neg_log_pvals_permuted_ols, _, _ = permuted_ols(
tested_var, target_vars, model_intercept=model_intercept,
n_perm=n_perm, two_sided_test=two_sided_test,
random_state=random_state, n_jobs=n_jobs, verbose=max(0, verbose - 1))
neg_log_corrected_pvals_img = masker.inverse_transform(
np.ravel(neg_log_pvals_permuted_ols))
return neg_log_corrected_pvals_img
class FirstLevelGLM(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for Single-session fMRI data
Parameters
----------
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters.
target_affine: 3x3 or 4x4 matrix, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
target_shape: 3-tuple of integers, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
low_pass: False or float, optional
This parameter is passed to nilearn.signal.clean.
Please see the related documentation for details.
high_pass: False or float, optional
This parameter is passed to nilearn.signal.clean.
Please see the related documentation for details.
t_r: float, optional
This parameter is passed to nilearn.signal.clean.
Please see the related documentation for details.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: instance of joblib.Memory or string
Used to cache the masking process.
By default, no caching is done. If a string is given, it is the
path to the caching directory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
standardize : boolean, optional
If standardize is True, the time-series are centered and normed:
their variance is put to 1 in the time dimension.
percent_signal_change: bool, optional,
If True, fMRI signals are scaled to percent of the mean value
Incompatible with standardize (standardize=False is enforced when\
percent_signal_change is True).
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
noise_model : {'ar1', 'ols'}, optional
the temporal variance model. Defaults to 'ar1'
Attributes
----------
labels : array of shape (n_voxels,),
a map of values on voxels used to identify the corresponding model
results : dict,
with keys corresponding to the different labels values
values are RegressionResults instances corresponding to the voxels
"""
def __init__(self,
mask=None,
target_affine=None,
target_shape=None,
low_pass=None,
high_pass=None,
t_r=None,
smoothing_fwhm=None,
memory=Memory(None),
memory_level=1,
standardize=False,
percent_signal_change=True,
verbose=1,
n_jobs=1,
noise_model='ar1'):
self.mask = mask
self.memory = memory
self.memory_level = memory_level
self.verbose = verbose
self.standardize = standardize
self.n_jobs = n_jobs
self.low_pass = low_pass
self.high_pass = high_pass
self.t_r = t_r
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
self.noise_model = noise_model
self.percent_signal_change = percent_signal_change
if self.percent_signal_change:
self.standardize = False
def fit(self, imgs, design_matrices):
""" Fit the GLM
1. does a masker job: fMRI_data -> Y
2. fit an ols regression to (Y, X)
3. fit an AR(1) regression of require
This results in an internal (labels_, regression_results_) parameters
Parameters
----------
imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/building_blocks/manipulating_mr_images.html#niimg.
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
design_matrices: pandas DataFrame or list of pandas DataFrames,
fMRI design matrices
"""
# First, learn the mask
if not isinstance(self.mask, NiftiMasker):
self.masker_ = NiftiMasker(mask_img=self.mask,
smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize,
low_pass=self.low_pass,
high_pass=self.high_pass,
mask_strategy='epi',
t_r=self.t_r,
memory=self.memory,
verbose=max(0, self.verbose - 1),
target_shape=self.target_shape,
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask)
for param_name in [
'target_affine', 'target_shape', 'smoothing_fwhm',
'low_pass', 'high_pass', 't_r', 'memory', 'memory_level'
]:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
# make design_matrices a list of arrays
if isinstance(design_matrices, (_basestring, pd.DataFrame)):
design_matrices_ = [design_matrices]
else:
design_matrices_ = [X for X in design_matrices]
design_matrices = []
for design_matrix in design_matrices_:
if isinstance(design_matrix, _basestring):
loaded = pd.read_csv(design_matrix, index_col=0)
design_matrices.append(loaded.values)
elif isinstance(design_matrix, pd.DataFrame):
design_matrices.append(design_matrix.values)
else:
raise TypeError(
'Design matrix can only be a pandas DataFrames or a'
'string. A %s was provided' % type(design_matrix))
# make imgs a list of Nifti1Images
if isinstance(imgs, (Nifti1Image, _basestring)):
imgs = [imgs]
if len(imgs) != len(design_matrices):
raise ValueError(
'len(imgs) %d does not match len(design_matrices) %d' %
(len(imgs), len(design_matrices)))
# Loop on imgs and design matrices
self.labels_, self.results_ = [], []
self.masker_.fit(imgs)
for X, img in zip(design_matrices, imgs):
Y = self.masker_.transform(img)
if self.percent_signal_change:
Y, _ = percent_mean_scaling(Y)
labels_, results_ = session_glm(Y,
X,
noise_model=self.noise_model,
bins=100)
self.labels_.append(labels_)
self.results_.append(results_)
return self
def transform(self,
con_vals,
contrast_type=None,
contrast_name='',
output_z=True,
output_stat=False,
output_effects=False,
output_variance=False):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
In multi-session case, outputs the fixed effects map.
Parameters
----------
con_vals : array or list of arrays of shape (n_col) or (n_dim, n_col)
where ``n_col`` is the number of columns of the design matrix,
numerical definition of the contrast (one array per run)
contrast_type : {'t', 'F'}, optional
type of the contrast
contrast_name : str, optional
name of the contrast
output_z : bool, optional
Return or not the corresponding z-stat image
output_stat : bool, optional
Return or not the base (t/F) stat image
output_effects : bool, optional
Return or not the corresponding effect image
output_variance : bool, optional
Return or not the corresponding variance image
Returns
-------
output_images : list of Nifti1Images
The desired output images
"""
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
if isinstance(con_vals, np.ndarray):
con_vals = [con_vals]
if len(con_vals) != len(self.results_):
raise ValueError(
'contrasts must be a sequence of %d session contrasts' %
len(self.results_))
contrast = None
for i, (labels_, results_, con_val) in enumerate(
zip(self.labels_, self.results_, con_vals)):
if np.all(con_val == 0):
warn('Contrast for session %d is null' % i)
contrast_ = compute_contrast(labels_, results_, con_val,
contrast_type)
if contrast is None:
contrast = contrast_
else:
contrast = contrast + contrast_
if output_z or output_stat:
# compute the contrast and stat
contrast.z_score()
# Prepare the returned images
do_outputs = [output_z, output_stat, output_effects, output_variance]
estimates = ['z_score_', 'stat_', 'effect', 'variance']
descrips = [
'z statistic', 'Statistical value', 'Estimated effect',
'Estimated variance'
]
output_images = []
for do_output, estimate, descrip in zip(do_outputs, estimates,
descrips):
if not do_output:
continue
estimate_ = getattr(contrast, estimate)
if estimate_.ndim == 3:
shape_ = estimate_.shape
estimate_ = np.reshape(estimate_,
(shape_[0] * shape_[1], shape_[2]))
output = self.masker_.inverse_transform(estimate_)
output.get_header()['descrip'] = ('%s of contrast %s' %
(descrip, contrast_name))
output_images.append(output)
return output_images
def fit_transform(self,
design_matrices,
fmri_images,
con_vals,
contrast_type=None,
contrast_name='',
output_z=True,
output_stat=False,
output_effects=False,
output_variance=False):
""" Fit then transform. For more details,
see FirstLevelGLM.fit and FirstLevelGLM.transform documentation"""
return self.fit(design_matrices,
fmri_images).transform(con_vals,
contrast_type,
contrast_name,
output_z=True,
output_stat=False,
output_effects=False,
output_variance=False)
"""
for USE_CENTROIDS in USE_CENTROIDS_SET:
for SPATIAL_NEIGHBORHOOD in SPATIAL_NEIGHBORHOOD_SET:
for N_CLUSTERS in N_CLUSTERS_SET:
if SPATIAL_NEIGHBORHOOD:
img_shape = masker.mask_img_.shape
x,y,z = np.meshgrid(np.arange(img_shape[1]),
np.arange(img_shape[0]),
np.arange(img_shape[2]))
x_img = nib.Nifti1Image(x, masker.affine_)
y_img = nib.Nifti1Image(y, masker.affine_)
z_img = nib.Nifti1Image(z, masker.affine_)
loc = masker.transform([x_img, y_img, z_img])
else:
loc = np.zeros((0, pet_data_masked.shape[1]))
##############################################################################
# MiniBatch Kmeans
##############################################################################
mbk = MiniBatchKMeans(init='k-means++', n_clusters=N_CLUSTERS,
n_init=10, max_no_improvement=10, verbose=0)
pet_loc_data = np.concatenate((pet_data_masked.T, loc.T), axis=1)
mbk.fit(pet_loc_data)
mbk_means_labels = mbk.labels_
contrasts = contrasts[1:]
elif task in ['HcpGambling', 'HcpLanguage', 'HcpRelational', 'HcpSocial']:
order = [1, 0]
contrasts = [contrasts[c] for c in order]
contrasts = contrasts[1:]
elif task in ['HcpMotor', 'HcpWm']:
order = [4, 3, 2, 1, 0]
contrasts = [contrasts[c] for c in order]
elif task == 'RSVPLanguage':
order = [7, 4, 0, 6, 3, 5, 1, 2]
contrasts = [contrasts[c] for c in order]
contrasts = contrasts[3:]
n_contrasts = len(contrasts)
for contrast in contrasts:
imgs = task_df[task_df.contrast == contrast].path.values
X = masker.transform(imgs)
correlations = append_correlation(imgs, masker, correlations)
correlations = np.array(correlations)
# Define subplot box for bar charts and its pos in the fig
ax = plt.axes(
[.3 + .48 * column, q * .048 + .115, .21, n_contrasts * .049])
ax.barh(np.arange(n_contrasts),
correlations.mean(1),
.75,
xerr=correlations.std(1),
error_kw=dict(capsize=2, captick=3),
color=colors[j])
ax.set_yticks(np.arange(n_contrasts))
new_labels = []
for i in range(n_contrasts):
if len(np.unique(haxby_labels)) != 9:
print "\n\n\n **** WARNING FOR SUBJECT %d (2) ***** \n\n\n" % si
# ## Find voxels of interest ##############################################
print("Build a mask based on the activations.")
epi_masker = NiftiMasker(mask_strategy='epi',
detrend=True,
standardize=True)
epi_masker = mem.cache(epi_masker.fit)(fmri_raw_img)
plot_roi(epi_masker.mask_img_,
bg_img=anat_img,
title='EPI mask (Subj %d)' % si)
print("Normalize the (transformed) data"
) # zscore per pixel, over examples.
fmri_masked_vectors = epi_masker.transform(fmri_raw_img)
fmri_normed_vectors = mem.cache(stats.mstats.zscore)(fmri_masked_vectors,
axis=0)
fmri_normed_img = epi_masker.inverse_transform(fmri_normed_vectors)
print("Smooth the (spatial) data.")
fmri_smooth_img = mem.cache(image.smooth_img)(fmri_normed_img, fwhm=6)
print("Mask the MRI data.")
masked_fmri_vectors = mem.cache(epi_masker.transform)(fmri_smooth_img)
fmri_masked_img = epi_masker.inverse_transform(masked_fmri_vectors)
# ## Use similarity across conditions as the 4th dimension ##########################################
print("Compute similarity via ttest.")
condition_names = list(np.unique(haxby_labels))
n_conds = len(condition_names)
def cluster_level_inference(stat_img,
mask_img=None,
threshold=3.,
alpha=.05,
verbose=False):
""" Report the proportion of active voxels for all clusters
defined by the input threshold.
Parameters
----------
stat_img : Niimg-like object or None, optional
statistical image (presumably in z scale)
mask_img : Niimg-like object, optional,
mask image
threshold: list of floats, optional
cluster-forming threshold in z-scale.
alpha: float or list, optional
level of control on the true positive rate, aka true dsicovery
proportion
verbose: bool, optional
verbosity mode
Returns
-------
proportion_true_discoveries_img: Nifti1Image,
the statistical map that gives the true positive
Note
----
This implements the method described in:
Rosenblatt JD, Finos L, Weeda WD, Solari A, Goeman JJ. All-Resolutions
Inference for brain imaging. Neuroimage. 2018 Nov 1;181:786-796. doi:
10.1016/j.neuroimage.2018.07.060
"""
if not isinstance(threshold, list):
threshold = [threshold]
if mask_img is None:
masker = NiftiMasker(mask_strategy='background').fit(stat_img)
else:
masker = NiftiMasker(mask_img=mask_img).fit()
stats = np.ravel(masker.transform(stat_img))
hommel_value = _compute_hommel_value(stats, alpha, verbose=verbose)
# embed it back to 3D grid
stat_map = get_data(masker.inverse_transform(stats))
# Extract connected components above threshold
proportion_true_discoveries_img = math_img('0. * img', img=stat_img)
proportion_true_discoveries = masker.transform(
proportion_true_discoveries_img).ravel()
for threshold_ in sorted(threshold):
label_map, n_labels = label(stat_map > threshold_)
labels = label_map[get_data(masker.mask_img_) > 0]
for label_ in range(1, n_labels + 1):
# get the z-vals in the cluster
cluster_vals = stats[labels == label_]
proportion = _true_positive_fraction(cluster_vals, hommel_value,
alpha)
proportion_true_discoveries[labels == label_] = proportion
proportion_true_discoveries_img = masker.inverse_transform(
proportion_true_discoveries)
return proportion_true_discoveries_img
def map_threshold(stat_img=None,
mask_img=None,
alpha=.001,
threshold=3.,
height_control='fpr',
cluster_threshold=0):
""" Compute the required threshold level and return the thresholded map
Parameters
----------
stat_img : Niimg-like object or None, optional
statistical image (presumably in z scale)
whenever height_control is 'fpr' or None,
stat_img=None is acceptable.
If it is 'fdr' or 'bonferroni', an error is raised if stat_img is None.
mask_img : Niimg-like object, optional,
mask image
alpha: float or list, optional
number controlling the thresholding (either a p-value or q-value).
Its actual meaning depends on the height_control parameter.
This function translates alpha to a z-scale threshold.
threshold: float, optional
desired threshold in z-scale.
This is used only if height_control is None
height_control: string, or None optional
false positive control meaning of cluster forming
threshold: 'fpr'|'fdr'|'bonferroni'\|None
cluster_threshold : float, optional
cluster size threshold. In the returned thresholded map,
sets of connected voxels (`clusters`) with size smaller
than this number will be removed.
Returns
-------
thresholded_map : Nifti1Image,
the stat_map thresholded at the prescribed voxel- and cluster-level
threshold: float,
the voxel-level threshold used actually
Note
----
If the input image is not z-scaled (i.e. some z-transformed statistic)
the computed threshold is not rigorous and likely meaningless
"""
height_control_methods = [
'fpr', 'fdr', 'bonferroni', 'all-resolution-inference', None
]
if height_control not in height_control_methods:
raise ValueError("height control should be one of {0}",
height_control_methods)
# if height_control is 'fpr' or None, we don't need to look at the data
# to compute the threshold
if height_control == 'fpr':
threshold = norm.isf(alpha)
# In this case, and if stat_img is None, we return
if stat_img is None:
if height_control in ['fpr', None]:
return None, threshold
else:
raise ValueError(
'Map_threshold requires stat_img not to be None'
'when the height_control procedure is "bonferroni" or "fdr"')
if mask_img is None:
masker = NiftiMasker(mask_strategy='background').fit(stat_img)
else:
masker = NiftiMasker(mask_img=mask_img).fit()
stats = np.ravel(masker.transform(stat_img))
n_voxels = np.size(stats)
if height_control == 'fdr':
threshold = fdr_threshold(stats, alpha)
elif height_control == 'bonferroni':
threshold = norm.isf(alpha / n_voxels)
stats *= (stats > threshold)
# embed it back to 3D grid
stat_map = get_data(masker.inverse_transform(stats))
# Extract connected components above threshold
label_map, n_labels = label(stat_map > threshold)
labels = label_map[get_data(masker.mask_img_) > 0]
for label_ in range(1, n_labels + 1):
if np.sum(labels == label_) < cluster_threshold:
stats[labels == label_] = 0
return masker.inverse_transform(stats), threshold
def map_threshold(stat_img, mask_img=None, threshold=.001,
height_control='fpr', cluster_threshold=0):
""" Threshold the provided map
Parameters
----------
stat_img : Niimg-like object,
statistical image (presumably in z scale)
mask_img : Niimg-like object, optional,
mask image
threshold: float, optional
cluster forming threshold (either a p-value or z-scale value)
height_control: string, optional
false positive control meaning of cluster forming
threshold: 'fpr'|'fdr'|'bonferroni'|'none'
cluster_threshold : float, optional
cluster size threshold
Returns
-------
thresholded_map : Nifti1Image,
the stat_map theresholded at the prescribed voxel- and cluster-level
threshold: float,
the voxel-level threshold used actually
"""
# Masking
if mask_img is None:
masker = NiftiMasker(mask_strategy='background').fit(stat_img)
else:
masker = NiftiMasker(mask_img=mask_img).fit()
stats = np.ravel(masker.transform(stat_img))
n_voxels = np.size(stats)
# Thresholding
if height_control == 'fpr':
z_th = norm.isf(threshold)
elif height_control == 'fdr':
z_th = fdr_threshold(stats, threshold)
elif height_control == 'bonferroni':
z_th = norm.isf(threshold / n_voxels)
else: # Brute-force thresholding
z_th = threshold
stats *= (stats > z_th)
# embed it back to 3D grid
stat_map = masker.inverse_transform(stats).get_data()
# Extract connected components above threshold
label_map, n_labels = label(stat_map > z_th)
labels = label_map[masker.mask_img_.get_data() > 0]
for label_ in range(1, n_labels + 1):
if np.sum(labels == label_) < cluster_threshold:
stats[labels == label_] = 0
return masker.inverse_transform(stats), z_th
class FirstLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for single session fMRI data
Parameters
----------
t_r : float
This parameter indicates repetition times of the experimental runs.
In seconds. It is necessary to correctly consider times in the design
matrix. This parameter is also passed to nilearn.signal.clean.
Please see the related documentation for details.
slice_time_ref : float, optional (default 0.)
This parameter indicates the time of the reference slice used in the
slice timing preprocessing step of the experimental runs. It is
expressed as a percentage of the t_r (time repetition), so it can have
values between 0. and 1.
hrf_model : {'spm', 'spm + derivative', 'spm + derivative + dispersion',
'glover', 'glover + derivative', 'glover + derivative + dispersion',
'fir', None}
String that specifies the hemodynamic response function. Defaults to 'glover'.
drift_model : string, optional
This parameter specifies the desired drift model for the design
matrices. It can be 'polynomial', 'cosine' or None.
period_cut : float, optional
This parameter specifies the cut period of the high-pass filter in
seconds for the design matrices. Used only if drift_model is 'cosine'.
drift_order : int, optional
This parameter specifices the order of the drift model (in case it is
polynomial) for the design matrices.
fir_delays : array of shape(n_onsets) or list, optional
In case of FIR design, yields the array of delays used in the FIR
model, in seconds.
min_onset : float, optional
This parameter specifies the minimal onset relative to the design
(in seconds). Events that start before (slice_time_ref * t_r +
min_onset) are not considered.
mask_img : Niimg-like, NiftiMasker object or False, optional
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a NiftiMasker with default
parameters. If False is given then the data will not be masked.
target_affine : 3x3 or 4x4 matrix, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
target_shape : 3-tuple of integers, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
smoothing_fwhm : float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory : string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level : integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
standardize : boolean, optional
If standardize is True, the time-series are centered and normed:
their variance is put to 1 in the time dimension.
signal_scaling : False, int or (int, int), optional,
If not False, fMRI signals are scaled to the mean value of scaling_axis
given, which can be 0, 1 or (0, 1). 0 refers to mean scaling each voxel
with respect to time, 1 refers to mean scaling each time point with
respect to all voxels and (0, 1) refers to scaling with respect to
voxels and time, which is known as grand mean scaling.
Incompatible with standardize (standardize=False is enforced when
signal_scaling is not False).
noise_model : {'ar1', 'ols'}, optional
The temporal variance model. Defaults to 'ar1'
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints progress by computation of
each run. If 2 prints timing details of masker and GLM. If 3
prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
subject_label : string, optional
This id will be used to identify a `FirstLevelModel` when passed to
a `SecondLevelModel` object.
Attributes
----------
labels : array of shape (n_voxels,),
a map of values on voxels used to identify the corresponding model
results : dict,
with keys corresponding to the different labels values
values are RegressionResults instances corresponding to the voxels
"""
@replace_parameters({'mask': 'mask_img'}, end_version='next')
def __init__(self, t_r=None, slice_time_ref=0., hrf_model='glover',
drift_model='cosine', period_cut=128, drift_order=1,
fir_delays=[0], min_onset=-24, mask_img=None, target_affine=None,
target_shape=None, smoothing_fwhm=None, memory=Memory(None),
memory_level=1, standardize=False, signal_scaling=0,
noise_model='ar1', verbose=0, n_jobs=1,
minimize_memory=True, subject_label=None):
# design matrix parameters
self.t_r = t_r
self.slice_time_ref = slice_time_ref
self.hrf_model = hrf_model
self.drift_model = drift_model
self.period_cut = period_cut
self.drift_order = drift_order
self.fir_delays = fir_delays
self.min_onset = min_onset
# glm parameters
self.mask_img = mask_img
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, _basestring):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.standardize = standardize
if signal_scaling is False:
self.signal_scaling = signal_scaling
elif signal_scaling in [0, 1, (0, 1)]:
self.scaling_axis = signal_scaling
self.signal_scaling = True
self.standardize = False
else:
raise ValueError('signal_scaling must be "False", "0", "1"'
' or "(0, 1)"')
self.noise_model = noise_model
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
# attributes
self.labels_ = None
self.results_ = None
self.subject_label = subject_label
def fit(self, run_imgs, events=None, confounds=None,
design_matrices=None):
""" Fit the GLM
For each run:
1. create design matrix X
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
run_imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/manipulating_images/input_output.html#inputing-data-file-names-or-image-objects
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
events: pandas Dataframe or string or list of pandas DataFrames or
strings
fMRI events used to build design matrices. One events object
expected per run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
confounds: pandas Dataframe or string or list of pandas DataFrames or
strings
Each column in a DataFrame corresponds to a confound variable
to be included in the regression model of the respective run_img.
The number of rows must match the number of volumes in the
respective run_img. Ignored in case designs is not None.
If string, then a path to a csv file is expected.
design_matrices: pandas DataFrame or list of pandas DataFrames,
Design matrices that will be used to fit the GLM. If given it
takes precedence over events and confounds.
"""
# Check arguments
# Check imgs type
if events is not None:
_check_events_file_uses_tab_separators(
events_files=events)
if not isinstance(run_imgs, (list, tuple)):
run_imgs = [run_imgs]
if design_matrices is None:
if events is None:
raise ValueError('events or design matrices must be provided')
if self.t_r is None:
raise ValueError('t_r not given to FirstLevelModel object'
' to compute design from events')
else:
design_matrices = _check_run_tables(run_imgs, design_matrices,
'design_matrices')
# Check that number of events and confound files match number of runs
# Also check that events and confound files can be loaded as DataFrame
if events is not None:
events = _check_run_tables(run_imgs, events, 'events')
if confounds is not None:
confounds = _check_run_tables(run_imgs, confounds, 'confounds')
# Learn the mask
if self.mask_img is False:
# We create a dummy mask to preserve functionality of api
ref_img = check_niimg(run_imgs[0])
self.mask_img = Nifti1Image(np.ones(ref_img.shape[:3]),
ref_img.affine)
if not isinstance(self.mask_img, NiftiMasker):
self.masker_ = NiftiMasker(
mask_img=self.mask_img, smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize, mask_strategy='epi',
t_r=self.t_r, memory=self.memory,
verbose=max(0, self.verbose - 2),
target_shape=self.target_shape,
memory_level=self.memory_level)
self.masker_.fit(run_imgs[0])
else:
if self.mask_img.mask_img_ is None and self.masker_ is None:
self.masker_ = clone(self.mask_img)
for param_name in ['target_affine', 'target_shape',
'smoothing_fwhm', 't_r', 'memory',
'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker'
' overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(run_imgs[0])
else:
self.masker_ = self.mask_img
# For each run fit the model and keep only the regression results.
self.labels_, self.results_, self.design_matrices_ = [], [], []
n_runs = len(run_imgs)
t0 = time.time()
for run_idx, run_img in enumerate(run_imgs):
# Report progress
if self.verbose > 0:
percent = float(run_idx) / n_runs
percent = round(percent * 100, 2)
dt = time.time() - t0
# We use a max to avoid a division by zero
if run_idx == 0:
remaining = 'go take a coffee, a big one'
else:
remaining = (100. - percent) / max(0.01, percent) * dt
remaining = '%i seconds remaining' % remaining
sys.stderr.write(
"Computing run %d out of %d runs (%s)\n"
% (run_idx + 1, n_runs, remaining))
# Build the experimental design for the glm
run_img = check_niimg(run_img, ensure_ndim=4)
if design_matrices is None:
n_scans = run_img.get_data().shape[3]
if confounds is not None:
confounds_matrix = confounds[run_idx].values
if confounds_matrix.shape[0] != n_scans:
raise ValueError('Rows in confounds does not match'
'n_scans in run_img at index %d'
% (run_idx,))
confounds_names = confounds[run_idx].columns.tolist()
else:
confounds_matrix = None
confounds_names = None
start_time = self.slice_time_ref * self.t_r
end_time = (n_scans - 1 + self.slice_time_ref) * self.t_r
frame_times = np.linspace(start_time, end_time, n_scans)
design = make_first_level_design_matrix(frame_times, events[run_idx],
self.hrf_model, self.drift_model,
self.period_cut, self.drift_order,
self.fir_delays, confounds_matrix,
confounds_names, self.min_onset)
else:
design = design_matrices[run_idx]
self.design_matrices_.append(design)
# Mask and prepare data for GLM
if self.verbose > 1:
t_masking = time.time()
sys.stderr.write('Starting masker computation \r')
Y = self.masker_.transform(run_img)
if self.verbose > 1:
t_masking = time.time() - t_masking
sys.stderr.write('Masker took %d seconds \n' % t_masking)
if self.signal_scaling:
Y, _ = mean_scaling(Y, self.scaling_axis)
if self.memory:
mem_glm = self.memory.cache(run_glm, ignore=['n_jobs'])
else:
mem_glm = run_glm
# compute GLM
if self.verbose > 1:
t_glm = time.time()
sys.stderr.write('Performing GLM computation\r')
labels, results = mem_glm(Y, design.values,
noise_model=self.noise_model,
bins=100, n_jobs=self.n_jobs)
if self.verbose > 1:
t_glm = time.time() - t_glm
sys.stderr.write('GLM took %d seconds \n' % t_glm)
self.labels_.append(labels)
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.results_.append(results)
del Y
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of %d runs done in %i seconds\n\n"
% (n_runs, time.time() - t0))
return self
def compute_contrast(self, contrast_def, stat_type=None,
output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
In multi-session case, outputs the fixed effects map.
Parameters
----------
contrast_def : str or array of shape (n_col) or list of (string or
array of shape (n_col))
where ``n_col`` is the number of columns of the design matrix,
(one array per run). If only one array is provided when there
are several runs, it will be assumed that the same contrast is
desired for all runs. The string can be a formula compatible with
the linear constraint of the Patsy library. Basically one can use
the name of the conditions as they appear in the design matrix of
the fitted model combined with operators /\*+- and numbers.
Please checks the patsy documentation for formula examples:
http://patsy.readthedocs.io/en/latest/API-reference.html#patsy.DesignInfo.linear_constraint
stat_type : {'t', 'F'}, optional
type of the contrast
output_type : str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size', 'effect_variance' or 'all'
Returns
-------
output : Nifti1Image or dict
The desired output image(s). If ``output_type == 'all'``, then
the output is a dictionary of images, keyed by the type of image.
"""
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
if isinstance(contrast_def, (np.ndarray, str)):
con_vals = [contrast_def]
elif isinstance(contrast_def, (list, tuple)):
con_vals = contrast_def
else:
raise ValueError('contrast_def must be an array or str or list of'
' (array or str)')
# Translate formulas to vectors with patsy
design_info = DesignInfo(self.design_matrices_[0].columns.tolist())
for cidx, con in enumerate(con_vals):
if not isinstance(con, np.ndarray):
con_vals[cidx] = design_info.linear_constraint(con).coefs
n_runs = len(self.labels_)
if len(con_vals) != n_runs:
warn('One contrast given, assuming it for all %d runs' % n_runs)
con_vals = con_vals * n_runs
# 'all' is assumed to be the final entry; if adding more, place before 'all'
valid_types = ['z_score', 'stat', 'p_value', 'effect_size',
'effect_variance', 'all']
if output_type not in valid_types:
raise ValueError('output_type must be one of {}'.format(valid_types))
contrast = _fixed_effect_contrast(self.labels_, self.results_,
con_vals, stat_type)
output_types = valid_types[:-1] if output_type == 'all' else [output_type]
outputs = {}
for output_type_ in output_types:
estimate_ = getattr(contrast, output_type_)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_vals)
output.header['descrip'] = (
'%s of contrast %s' % (output_type_, contrast_name))
outputs[output_type_] = output
return outputs if output_type == 'all' else output
y_test = target[condition_mask_test]
y_train[y_train=='scissors']=1
y_train[y_train=='scrambledpix']=-1
y_train=np.array(y_train.astype('double'))
y_test[y_test=='scissors']=1
y_test[y_test=='scrambledpix']=-1
y_test=np.array(y_test.astype('double'))
masker = NiftiMasker(mask_strategy='epi',standardize=True)
X_train = masker.fit_transform(X_train)
X_test = masker.transform(X_test)
mask = masker.mask_img_.get_data().astype(np.bool)
mask= _crop_mask(mask)
background_img = mean_img(data_files.func[0])
X_train, y_train, _, y_train_mean, _ = center_data(X_train, y_train, fit_intercept=True, normalize=False,copy=False)
X_test-=X_train.mean(axis=0)
X_test/=np.std(X_train,axis=0)
alpha=1
ratio=0.5
k=200
solver_params = dict(tol=1e-6, max_iter=5000,prox_max_iter=100)
def map_threshold(stat_img=None, mask_img=None, level=.001,
height_control='fpr', cluster_threshold=0):
""" Compute the required threshold level and return the thresholded map
Parameters
----------
stat_img : Niimg-like object or None, optional
statistical image (presumably in z scale)
whenever height_control is 'fpr' or None,
stat_img=None is acceptable.
If it is 'fdr' or 'bonferroni', an error is raised if stat_img is None.
mask_img : Niimg-like object, optional,
mask image
level: float, optional
number controling the thresholding (either a p-value or z-scale value).
Not to be confused with the z-scale threshold: level can be a p-values,
e.g. "0.05" or another type of number depending on the
height_control parameter. The z-scale threshold is actually returned by
the function.
height_control: string, or None optional
false positive control meaning of cluster forming
threshold: 'fpr'|'fdr'|'bonferroni'\|None
cluster_threshold : float, optional
cluster size threshold. In the returned thresholded map,
sets of connected voxels (`clusters`) with size smaller
than this number will be removed.
Returns
-------
thresholded_map : Nifti1Image,
the stat_map thresholded at the prescribed voxel- and cluster-level
threshold: float,
the voxel-level threshold used actually
Note
----
If the input image is not z-scaled (i.e. some z-transformed statistic)
the computed threshold is not rigorous and likely meaningless
"""
# Check that height_control is correctly specified
if height_control not in ['fpr', 'fdr', 'bonferroni', None]:
raise ValueError(
"height control should be one of ['fpr', 'fdr', 'bonferroni', None]")
# if height_control is 'fpr' or None, we don't need to look at the data
# to compute the threhsold
if height_control == 'fpr':
threshold = norm.isf(level)
elif height_control is None:
threshold = level
# In this case, and is stat_img is None, we return
if stat_img is None:
if height_control in ['fpr', None]:
return None, threshold
else:
raise ValueError(
'Map_threshold requires stat_img not to be None'
'when the heigh_control procedure is bonferroni or fdr')
# Masking
if mask_img is None:
masker = NiftiMasker(mask_strategy='background').fit(stat_img)
else:
masker = NiftiMasker(mask_img=mask_img).fit()
stats = np.ravel(masker.transform(stat_img))
n_voxels = np.size(stats)
# Thresholding
if height_control == 'fdr':
threshold = fdr_threshold(stats, level)
elif height_control == 'bonferroni':
threshold = norm.isf(level / n_voxels)
stats *= (stats > threshold)
# embed it back to 3D grid
stat_map = masker.inverse_transform(stats).get_data()
# Extract connected components above threshold
label_map, n_labels = label(stat_map > threshold)
labels = label_map[masker.mask_img_.get_data() > 0]
for label_ in range(1, n_labels + 1):
if np.sum(labels == label_) < cluster_threshold:
stats[labels == label_] = 0
return masker.inverse_transform(stats), threshold
labels = np.int32(labels)
# contrasts are IN ORDER -> shuffle!
new_inds = np.arange(0, X_task.shape[0])
np.random.shuffle(new_inds)
X_task = X_task[new_inds]
labels = labels[new_inds]
# subs = subs[new_inds]
# rest
# X_rest = nifti_masker.transform('preload_HR20persub_10mm_ero2.nii')
# X_rest = nifti_masker.transform('dump_rs_spca_s12_tmp')
rs_spca_data = joblib.load('dump_rs_spca_s12_tmp')
rs_spca_niis = nib.Nifti1Image(rs_spca_data,
nifti_masker.mask_img_.get_affine())
X_rest = nifti_masker.transform(rs_spca_niis)
del rs_spca_niis
del rs_spca_data
X_task = StandardScaler().fit_transform(X_task)
X_rest = StandardScaler().fit_transform(X_rest)
# ARCHI task
AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm')
AT_X = nifti_masker.transform(AT_niis)
AT_X = StandardScaler().fit_transform(AT_X)
print('done :)')
##############################################################################
# define computation graph
##############################################################################
class SecondLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for multiple subject
fMRI data
Parameters
----------
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters. Automatic mask computation assumes first level imgs have
already been masked.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
If 0 prints nothing. If 1 prints final computation time.
If 2 prints masker computation details.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
"""
def __init__(self,
mask=None,
smoothing_fwhm=None,
memory=None,
memory_level=1,
verbose=0,
n_jobs=1,
minimize_memory=True):
self.mask = mask
self.smoothing_fwhm = smoothing_fwhm
self.memory = memory
self.memory_level = memory_level
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
self.second_level_input_ = None
self.confounds_ = None
def fit(self, second_level_input, confounds=None, design_matrix=None):
""" Fit the second-level GLM
1. create design matrix
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
second_level_input: list of `FirstLevelModel` objects or pandas
DataFrame or list of Niimg-like objects.
Giving FirstLevelModel objects will allow to easily compute
the second level contast of arbitrary first level contrasts thanks
to the first_level_contrast argument of the compute_contrast
method. Effect size images will be computed for each model to
contrast at the second level.
If a pandas DataFrame, then they have to contain subject_label,
map_name and effects_map_path. It can contain multiple maps that
would be selected during contrast estimation with the argument
first_level_contrast of the compute_contrast function. The
DataFrame will be sorted based on the subject_label column to avoid
order inconsistencies when extracting the maps. So the rows of the
automatically computed design matrix, if not provided, will
correspond to the sorted subject_label column.
If list of Niimg-like objects then this is taken literally as Y
for the model fit and design_matrix must be provided.
confounds: pandas DataFrame, optional
Must contain a subject_label column. All other columns are
considered as confounds and included in the model. If
design_matrix is provided then this argument is ignored.
The resulting second level design matrix uses the same column
names as in the given DataFrame for confounds. At least two columns
are expected, "subject_label" and at least one confound.
design_matrix: pandas DataFrame, optional
Design matrix to fit the GLM. The number of rows
in the design matrix must agree with the number of maps derived
from second_level_input.
Ensure that the order of maps given by a second_level_input
list of Niimgs matches the order of the rows in the design matrix.
Must contain a column of 1s with column name 'intercept'.
"""
# Check parameters
# check first level input
if isinstance(second_level_input, list):
if len(second_level_input) < 2:
raise ValueError('A second level model requires a list with at'
'least two first level models or niimgs')
# Check FirstLevelModel objects case
if isinstance(second_level_input[0], FirstLevelModel):
models_input = enumerate(second_level_input)
for model_idx, first_level_model in models_input:
if (first_level_model.labels_ is None
or first_level_model.results_ is None):
raise ValueError(
'Model %s at index %i has not been fit yet'
'' % (first_level_model.subject_label, model_idx))
if not isinstance(first_level_model, FirstLevelModel):
raise ValueError(' object at idx %d is %s instead of'
' FirstLevelModel object' %
(model_idx, type(first_level_model)))
if confounds is not None:
if first_level_model.subject_label is None:
raise ValueError(
'In case confounds are provided, first level '
'objects need to provide the attribute '
'subject_label to match rows appropriately.'
'Model at idx %d does not provide it. '
'To set it, you can do '
'first_level_model.subject_label = "01"'
'' % (model_idx))
# Check niimgs case
elif isinstance(second_level_input[0], (str, Nifti1Image)):
if design_matrix is None:
raise ValueError('List of niimgs as second_level_input'
' require a design matrix to be provided')
for model_idx, niimg in enumerate(second_level_input):
if not isinstance(niimg, (str, Nifti1Image)):
raise ValueError(' object at idx %d is %s instead of'
' Niimg-like object' %
(model_idx, type(niimg)))
# Check pandas dataframe case
elif isinstance(second_level_input, pd.DataFrame):
for col in ['subject_label', 'map_name', 'effects_map_path']:
if col not in second_level_input.columns:
raise ValueError('second_level_input DataFrame must have'
' columns subject_label, map_name and'
' effects_map_path')
# Make sure subject_label contain strings
second_level_columns = second_level_input.columns.tolist()
labels_index = second_level_columns.index('subject_label')
labels_dtype = second_level_input.dtypes[labels_index]
if not isinstance(labels_dtype, np.object):
raise ValueError('subject_label column must be of dtype '
'object instead of dtype %s' % labels_dtype)
else:
raise ValueError('second_level_input must be a list of'
' `FirstLevelModel` objects, a pandas DataFrame'
' or a list Niimg-like objects. Instead %s '
'was provided' % type(second_level_input))
# check confounds
if confounds is not None:
if not isinstance(confounds, pd.DataFrame):
raise ValueError('confounds must be a pandas DataFrame')
if 'subject_label' not in confounds.columns:
raise ValueError('confounds DataFrame must contain column'
'"subject_label"')
if len(confounds.columns) < 2:
raise ValueError('confounds should contain at least 2 columns'
'one called "subject_label" and the other'
'with a given confound')
# Make sure subject_label contain strings
labels_index = confounds.columns.tolist().index('subject_label')
labels_dtype = confounds.dtypes[labels_index]
if not isinstance(labels_dtype, np.object):
raise ValueError('subject_label column must be of dtype '
'object instead of dtype %s' % labels_dtype)
# check design matrix
if design_matrix is not None:
if not isinstance(design_matrix, pd.DataFrame):
raise ValueError('design matrix must be a pandas DataFrame')
if 'intercept' not in design_matrix.columns:
raise ValueError('design matrix must contain "intercept"')
# sort a pandas dataframe by subject_label to avoid inconsistencies
# with the design matrix row order when automatically extracting maps
if isinstance(second_level_input, pd.DataFrame):
# Avoid pandas df.sort_value to keep compatibility with numpy 1.8
# also pandas df.sort since it is completely deprecated.
columns = second_level_input.columns.tolist()
column_index = columns.index('subject_label')
sorted_matrix = sorted(second_level_input.as_matrix(),
key=lambda x: x[column_index])
sorted_input = pd.DataFrame(sorted_matrix, columns=columns)
second_level_input = sorted_input
self.second_level_input_ = second_level_input
self.confounds_ = confounds
# Report progress
t0 = time.time()
if self.verbose > 0:
sys.stderr.write("Fitting second level model. "
"Take a deep breath\r")
# Select sample map for masker fit and get subjects_label for design
if isinstance(second_level_input, pd.DataFrame):
sample_map = second_level_input['effects_map_path'][0]
labels = second_level_input['subject_label']
subjects_label = labels.values.tolist()
elif isinstance(second_level_input[0], FirstLevelModel):
sample_model = second_level_input[0]
sample_condition = sample_model.design_matrices_[0].columns[0]
sample_map = sample_model.compute_contrast(
sample_condition, output_type='effect_size')
labels = [model.subject_label for model in second_level_input]
subjects_label = labels
else:
# In this case design matrix had to be provided
sample_map = second_level_input[0]
# Create and set design matrix, if not given
if design_matrix is None:
design_matrix = create_second_level_design(subjects_label,
confounds)
self.design_matrix_ = design_matrix
# Learn the mask. Assume the first level imgs have been masked.
if not isinstance(self.mask, NiftiMasker):
self.masker_ = NiftiMasker(mask_img=self.mask,
smoothing_fwhm=self.smoothing_fwhm,
memory=self.memory,
verbose=max(0, self.verbose - 1),
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask)
for param_name in ['smoothing_fwhm', 'memory', 'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(sample_map)
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of second level model done in "
"%i seconds\n" % (time.time() - t0))
return self
def compute_contrast(self,
second_level_contrast='intercept',
first_level_contrast=None,
second_level_stat_type=None,
output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
Parameters
----------
second_level_contrast: str or array of shape (n_col), optional
Where ``n_col`` is the number of columns of the design matrix,
The string can be a formula compatible with the linear constraint
of the Patsy library. Basically one can use the name of the
conditions as they appear in the design matrix of
the fitted model combined with operators /*+- and numbers.
Please check the patsy documentation for formula examples:
http://patsy.readthedocs.io/en/latest/API-reference.html#patsy.DesignInfo.linear_constraint
VERY IMPORTANT: The 'intercept' corresponds to the second level
effect after taking confounders in consideration. If there are
no confounders then this will be equivalent to a simple t test.
By default we compute the 'intercept' second level contrast.
first_level_contrast: str or array of shape (n_col) with respect to
FirstLevelModel, optional
In case a list of FirstLevelModel was provided as
second_level_input, we have to provide a contrast to apply to
the first level models to get the corresponding list of images
desired, that would be tested at the second level. In case a
pandas DataFrame was provided as second_level_input this is the
map name to extract from the pandas dataframe map_name column.
It has to be a 't' contrast.
second_level_stat_type: {'t', 'F'}, optional
Type of the second level contrast
output_type: str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size' or 'effect_variance'
Returns
-------
output_image: Nifti1Image
The desired output image
"""
if self.second_level_input_ is None:
raise ValueError('The model has not been fit yet')
# first_level_contrast check
if isinstance(self.second_level_input_[0], FirstLevelModel):
if first_level_contrast is None:
raise ValueError('If second_level_input was a list of '
'FirstLevelModel, then first_level_contrast '
'is mandatory. It corresponds to the '
'second_level_contrast argument of the '
'compute_contrast method of FirstLevelModel')
# check contrast definition
if isinstance(second_level_contrast, np.ndarray):
con_val = second_level_contrast
if np.all(con_val == 0):
raise ValueError('Contrast is null')
else:
design_info = DesignInfo(self.design_matrix_.columns.tolist())
constraint = design_info.linear_constraint(second_level_contrast)
con_val = constraint.coefs
# check output type
if isinstance(output_type, _basestring):
if output_type not in [
'z_score', 'stat', 'p_value', 'effect_size',
'effect_variance'
]:
raise ValueError(
'output_type must be one of "z_score", "stat"'
', "p_value", "effect_size" or "effect_variance"')
else:
raise ValueError('output_type must be one of "z_score", "stat",'
' "p_value", "effect_size" or "effect_variance"')
# Get effect_maps appropriate for chosen contrast
effect_maps = _infer_effect_maps(self.second_level_input_,
first_level_contrast)
# check design matrix X and effect maps Y agree on number of rows
if len(effect_maps) != self.design_matrix_.shape[0]:
raise ValueError(
'design_matrix does not match the number of maps considered. '
'%i rows in design matrix do not match with %i maps' %
(self.design_matrix_.shape[0], len(effect_maps)))
# Fit an OLS regression for parametric statistics
Y = self.masker_.transform(effect_maps)
if self.memory is not None:
arg_ignore = ['n_jobs']
mem_glm = self.memory.cache(run_glm, ignore=arg_ignore)
else:
mem_glm = run_glm
labels, results = mem_glm(Y,
self.design_matrix_.as_matrix(),
n_jobs=self.n_jobs,
noise_model='ols')
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.labels_ = labels
self.results_ = results
# We compute contrast object
if self.memory is not None:
mem_contrast = self.memory.cache(compute_contrast)
else:
mem_contrast = compute_contrast
contrast = mem_contrast(self.labels_, self.results_, con_val,
second_level_stat_type)
# We get desired output from contrast object
estimate_ = getattr(contrast, output_type)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
contrast_name = str(con_val)
output.header['descrip'] = ('%s of contrast %s' %
(output_type, contrast_name))
return output
prior_group)
meta_files = [path.replace('~', project_dir) for path in meta_files_pathes]
meta_categories = [
nifti_path.split(os.sep)[-1].split('_')[-1].split('.nii')[0]
for nifti_path in meta_files
]
print 'Found %i %s meta priors' % (len(meta_files), prior_group)
# concatenate priors
GM_masker = NiftiMasker(project_dir + '/data/raw/ref_vbm_bin.nii.gz')
GM_masker.fit()
prior_voxels = []
meta_check1 = []
for meta_file in meta_files:
prior = nib.load(meta_file)
prior_masked = GM_masker.transform(prior)
prior_voxels.append(prior_masked[0])
meta_check1.append(meta_file)
prior_array = np.array(prior_voxels)
# process prior_space
if method == 'ha_pc':
new_prior_array = copy.deepcopy(prior_array)
elif method == 'md_pc':
new_prior_array = prior_array - np.mean(prior_array, axis=0)
elif method == 'ts_pc':
new_prior_array = preprocessing.StandardScaler().fit_transform(
prior_array)
# extract voxel maps for each prior and k
for k in str(k_list).split(','):
def compute_fixed_effects(contrast_imgs, variance_imgs, mask=None,
precision_weighted=False):
"""Compute the fixed effects, given images of effects and variance
Parameters
----------
contrast_imgs : list of Nifti1Images or strings
The input contrast images.
variance_imgs : list of Nifti1Images or strings
The input variance images.
mask : Nifti1Image or NiftiMasker instance or None, optional
Mask image. If None, it is recomputed from contrast_imgs.
precision_weighted : Bool, optional
Whether fixed effects estimates should be weighted by inverse
variance or not. Default=False.
Returns
-------
fixed_fx_contrast_img : Nifti1Image
The fixed effects contrast computed within the mask.
fixed_fx_variance_img : Nifti1Image
The fixed effects variance computed within the mask.
fixed_fx_t_img : Nifti1Image
The fixed effects t-test computed within the mask.
Notes
-----
This function is experimental.
It may change in any future release of Nilearn.
"""
if len(contrast_imgs) != len(variance_imgs):
raise ValueError(
'The number of contrast images (%d) '
'differs from the number of variance images (%d). '
% (len(contrast_imgs), len(variance_imgs))
)
if isinstance(mask, NiftiMasker):
masker = mask.fit()
elif mask is None:
masker = NiftiMasker().fit(contrast_imgs)
else:
masker = NiftiMasker(mask_img=mask).fit()
variances = masker.transform(variance_imgs)
contrasts = masker.transform(contrast_imgs)
(fixed_fx_contrast,
fixed_fx_variance, fixed_fx_t) = _compute_fixed_effects_params(
contrasts, variances, precision_weighted)
fixed_fx_contrast_img = masker.inverse_transform(fixed_fx_contrast)
fixed_fx_variance_img = masker.inverse_transform(fixed_fx_variance)
fixed_fx_t_img = masker.inverse_transform(fixed_fx_t)
return fixed_fx_contrast_img, fixed_fx_variance_img, fixed_fx_t_img
masker.fit()
n_vox = r_mask.get_data().sum()
n_files = len(nii_paths)
if op.exists('dump_FS.npy'):
FS = np.load('dump_FS.npy')
cond_labels = np.load('dump_cond.npy')
sub_labels = np.load('dump_subs.npy')
else:
FS = np.zeros((n_files, n_vox))
for i_nii in range(n_files):
print('Loading nifti into memory: %i/%i' % (i_nii + 1, n_files))
data = np.nan_to_num(nib.load(nii_paths[i_nii]).get_data())
nii = nib.Nifti1Image(data, r_mask.get_affine())
cur_1d_data = masker.transform(nii)
FS[i_nii, :] = cur_1d_data
# save feature space to disk
np.save('dump_FS', FS)
np.save('dump_cond', cond_labels)
np.save('dump_subs', sub_labels)
from sklearn.preprocessing import StandardScaler
FS = StandardScaler().fit_transform(FS)
labels = cond_labels
# type conversion
FS = np.float32(FS)
labels = np.int32(labels)
mask_img = nifti_masker.mask_img_ ### Visualize the mask ######################################################## import matplotlib.pyplot as plt from nilearn.plotting import plot_roi from nilearn.image.image import mean_img # calculate mean image for the background mean_func_img = mean_img(func_filename) plot_roi(mask_img, mean_func_img, display_mode='y', cut_coords=4, title="Mask") ### Preprocess data ########################################################### nifti_masker.fit(func_filename) fmri_masked = nifti_masker.transform(func_filename) ### Run an algorithm ########################################################## from sklearn.decomposition import FastICA n_components = 20 ica = FastICA(n_components=n_components, random_state=42) components_masked = ica.fit_transform(fmri_masked.T).T ### Reverse masking ########################################################### components = nifti_masker.inverse_transform(components_masked) ### Show results ############################################################## from nilearn.plotting import plot_stat_map from nilearn.image import index_img plot_stat_map(index_img(components, 0), mean_func_img, display_mode='y',
target_header = masknii.get_header()
target_shape = masknii.shape
nifti_masker = NiftiMasker(mask_img=masknii, smoothing_fwhm=False,
standardize=False)
nifti_masker.fit()
nifti_masker.mask_img_.to_filename("debug_mask.nii")
for compr_name in ['PCA', 'FactorAnalysis',
'FastICA', 'SparsePCA']:
for sample in ['AT', 'HT']:
FS_nii, labels, subs = joblib.load('preload_2nd_' + sample)
print('%s in %s' % (compr_name, sample))
FS = nifti_masker.transform(FS_nii)
compressor = joblib.load('preload_compr_HT%s40' % compr_name)
FS_loadings = compressor.transform(FS)
from scipy.stats import f_oneway
for group, group_name in zip([labels, subs],
['tasks', 'subjects']):
print(group_name)
array_form = [FS_loadings[group_tag == group].ravel() for group_tag in np.unique(group)]
fvalue, pvalue = f_oneway(*array_form)
print('F-Value: %.4f' % fvalue)
print('P-Value: %.16f' % pvalue)
class FirstLevelModel(BaseEstimator, TransformerMixin, CacheMixin):
""" Implementation of the General Linear Model for single session fMRI data
Parameters
----------
t_r: float
This parameter indicates repetition times of the experimental runs.
In seconds. It is necessary to correctly consider times in the design
matrix. This parameter is also passed to nilearn.signal.clean.
Please see the related documentation for details.
slice_time_ref: float, optional (default 0.)
This parameter indicates the time of the reference slice used in the
slice timing preprocessing step of the experimental runs. It is
expressed as a percentage of the t_r (time repetition), so it can have
values between 0. and 1.
hrf_model : string, optional
This parameter specifies the hemodynamic response function (HRF) for
the design matrices. It can be 'canonical', 'canonical with derivative'
or 'fir'.
drift_model : string, optional
This parameter specifies the desired drift model for the design
matrices. It can be 'polynomial', 'cosine' or 'blank'.
period_cut : float, optional
This parameter specifies the cut period of the low-pass filter in
seconds for the design matrices.
drift_order : int, optional
This parameter specifices the order of the drift model (in case it is
polynomial) for the design matrices.
fir_delays : array of shape(n_onsets) or list, optional
In case of FIR design, yields the array of delays used in the FIR
model, in seconds.
min_onset : float, optional
This parameter specifies the minimal onset relative to the design
(in seconds). Events that start before (slice_time_ref * t_r +
min_onset) are not considered.
mask: Niimg-like, NiftiMasker or MultiNiftiMasker object, optional,
Mask to be used on data. If an instance of masker is passed,
then its mask will be used. If no mask is given,
it will be computed automatically by a MultiNiftiMasker with default
parameters.
target_affine: 3x3 or 4x4 matrix, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
target_shape: 3-tuple of integers, optional
This parameter is passed to nilearn.image.resample_img. Please see the
related documentation for details.
smoothing_fwhm: float, optional
If smoothing_fwhm is not None, it gives the size in millimeters of the
spatial smoothing to apply to the signal.
memory: string, optional
Path to the directory used to cache the masking process and the glm
fit. By default, no caching is done. Creates instance of joblib.Memory.
memory_level: integer, optional
Rough estimator of the amount of memory used by caching. Higher value
means more memory for caching.
standardize : boolean, optional
If standardize is True, the time-series are centered and normed:
their variance is put to 1 in the time dimension.
signal_scaling: False, int or (int, int), optional,
If not False, fMRI signals are scaled to the mean value of scaling_axis
given, which can be 0, 1 or (0, 1). 0 refers to mean scaling each voxel
with respect to time, 1 refers to mean scaling each time point with
respect to all voxels and (0, 1) refers to scaling with respect to
voxels and time, which is known as grand mean scaling.
Incompatible with standardize (standardize=False is enforced when
signal_scaling is not False).
noise_model : {'ar1', 'ols'}, optional
The temporal variance model. Defaults to 'ar1'
verbose : integer, optional
Indicate the level of verbosity. By default, nothing is printed.
n_jobs : integer, optional
The number of CPUs to use to do the computation. -1 means
'all CPUs', -2 'all CPUs but one', and so on.
minimize_memory : boolean, optional
Gets rid of some variables on the model fit results that are not
necessary for contrast computation and would only be useful for
further inspection of model details. This has an important impact
on memory consumption. True by default.
Attributes
----------
labels : array of shape (n_voxels,),
a map of values on voxels used to identify the corresponding model
results : dict,
with keys corresponding to the different labels values
values are RegressionResults instances corresponding to the voxels
"""
def __init__(self, t_r=None, slice_time_ref=0., hrf_model='glover',
drift_model='cosine', period_cut=128, drift_order=1,
fir_delays=[0], min_onset=-24, mask=None, target_affine=None,
target_shape=None, smoothing_fwhm=None, memory=Memory(None),
memory_level=1, standardize=False, signal_scaling=0,
noise_model='ar1', verbose=1, n_jobs=1,
minimize_memory=True):
# design matrix parameters
self.t_r = t_r
self.slice_time_ref = slice_time_ref
self.hrf_model = hrf_model
self.drift_model = drift_model
self.period_cut = period_cut
self.drift_order = drift_order
self.fir_delays = fir_delays
self.min_onset = min_onset
# glm parameters
self.mask = mask
self.target_affine = target_affine
self.target_shape = target_shape
self.smoothing_fwhm = smoothing_fwhm
if isinstance(memory, _basestring):
self.memory = Memory(memory)
else:
self.memory = memory
self.memory_level = memory_level
self.standardize = standardize
if signal_scaling in [0, 1, (0, 1)]:
self.scaling_axis = signal_scaling
self.signal_scaling = True
self.standardize = False
elif signal_scaling is False:
self.signal_scaling = signal_scaling
else:
raise ValueError('signal_scaling must be "False", "0", "1"'
' or "(0, 1)"')
self.noise_model = noise_model
self.verbose = verbose
self.n_jobs = n_jobs
self.minimize_memory = minimize_memory
# attributes
self.labels_ = None
self.results_ = None
def fit(self, run_imgs, paradigms=None, confounds=None,
design_matrices=None):
""" Fit the GLM
For each run:
1. create design matrix X
2. do a masker job: fMRI_data -> Y
3. fit regression to (Y, X)
Parameters
----------
run_imgs: Niimg-like object or list of Niimg-like objects,
See http://nilearn.github.io/building_blocks/manipulating_mr_images.html#niimg.
Data on which the GLM will be fitted. If this is a list,
the affine is considered the same for all.
paradigms: pandas Dataframe or string or list of pandas DataFrames or
strings,
fMRI paradigms used to build design matrices. One paradigm expected
per run_img. Ignored in case designs is not None.
confounds: pandas Dataframe or string or list of pandas DataFrames or
strings,
Each column in a DataFrame corresponds to a confound variable
to be included in the regression model of the respective run_img.
The number of rows must match the number of volumes in the
respective run_img. Ignored in case designs is not None.
design_matrices: pandas DataFrame or list of pandas DataFrames,
Design matrices that will be used to fit the GLM.
"""
# Check arguments
# Check imgs type
if not isinstance(run_imgs, (list, tuple)):
run_imgs = [run_imgs]
for rimg in run_imgs:
if not isinstance(rimg, (_basestring, Nifti1Image)):
raise ValueError('run_imgs must be Niimg-like object or list'
' of Niimg-like objects')
# check all information necessary to build design matrices is available
if design_matrices is None:
if paradigms is None:
raise ValueError('paradigms or design matrices must be provided')
if self.t_r is None:
raise ValueError('t_r not given to FirstLevelModel object'
' to compute design from paradigm')
else:
design_matrices = _check_run_tables(run_imgs, design_matrices,
'design_matrices')
# check the number of paradigm and confound files match number of runs
# Also check paradigm and confound files can be loaded as DataFrame
if paradigms is not None:
paradigms = _check_run_tables(run_imgs, paradigms, 'paradigms')
if confounds is not None:
confounds = _check_run_tables(run_imgs, confounds, 'confounds')
# Learn the mask
if not isinstance(self.mask, NiftiMasker):
self.masker_ = NiftiMasker(
mask_img=self.mask, smoothing_fwhm=self.smoothing_fwhm,
target_affine=self.target_affine,
standardize=self.standardize, mask_strategy='epi',
t_r=self.t_r, memory=self.memory,
verbose=max(0, self.verbose - 1),
target_shape=self.target_shape,
memory_level=self.memory_level)
else:
self.masker_ = clone(self.mask)
for param_name in ['target_affine', 'target_shape',
'smoothing_fwhm', 'low_pass', 'high_pass',
't_r', 'memory', 'memory_level']:
our_param = getattr(self, param_name)
if our_param is None:
continue
if getattr(self.masker_, param_name) is not None:
warn('Parameter %s of the masker overriden' % param_name)
setattr(self.masker_, param_name, our_param)
self.masker_.fit(run_imgs[0])
# For each run fit the model and keep only the regression results.
self.labels_, self.results_, self.design_matrices_ = [], [], []
n_runs = len(run_imgs)
t0 = time.time()
for run_idx, run_img in enumerate(run_imgs):
# Report progress
if self.verbose > 0:
percent = float(run_idx) / n_runs
percent = round(percent * 100, 2)
dt = time.time() - t0
# We use a max to avoid a division by zero
if run_idx == 0:
remaining = 'go take a coffee, a big one'
else:
remaining = (100. - percent) / max(0.01, percent) * dt
remaining = '%i seconds remaining' % remaining
sys.stderr.write(" " * 100 + "\r")
sys.stderr.write(
"Computing run %d out of %d runs (%s)\r"
% (run_idx, n_runs, remaining))
# Build the experimental design for the glm
run_img = check_niimg(run_img, ensure_ndim=4)
if design_matrices is None:
n_scans = run_img.get_data().shape[3]
if confounds is not None:
confounds_matrix = confounds[run_idx].values
if confounds_matrix.shape[0] != n_scans:
raise ValueError('Rows in confounds does not match'
'n_scans in run_img at index %d'
% (run_idx,))
confounds_names = confounds[run_idx].columns
else:
confounds_matrix = None
confounds_names = None
start_time = self.slice_time_ref * self.t_r
end_time = (n_scans - 1 + self.slice_time_ref) * self.t_r
frame_times = np.linspace(start_time, end_time, n_scans)
design = make_design_matrix(frame_times, paradigms[run_idx],
self.hrf_model, self.drift_model,
self.period_cut, self.drift_order,
self.fir_delays, confounds_matrix,
confounds_names, self.min_onset)
else:
design = design_matrices[run_idx]
self.design_matrices_.append(design)
# Compute GLM
Y = self.masker_.transform(run_img)
if self.signal_scaling:
Y, _ = mean_scaling(Y, self.scaling_axis)
if self.memory is not None:
mem_glm = self.memory.cache(run_glm)
else:
mem_glm = run_glm
labels, results = mem_glm(Y, design,
noise_model=self.noise_model,
bins=100, n_jobs=self.n_jobs)
self.labels_.append(labels)
# We save memory if inspecting model details is not necessary
if self.minimize_memory:
for key in results:
results[key] = SimpleRegressionResults(results[key])
self.results_.append(results)
del Y
# Report progress
if self.verbose > 0:
sys.stderr.write("\nComputation of %d runs done in %i seconds\n"
% (n_runs, time.time() - t0))
return self
def compute_contrast(self, contrast_def, contrast_name=None,
stat_type=None, output_type='z_score'):
"""Generate different outputs corresponding to
the contrasts provided e.g. z_map, t_map, effects and variance.
In multi-session case, outputs the fixed effects map.
Parameters
----------
contrast_def : array or list of arrays of shape (n_col) or (n_run, n_col)
where ``n_col`` is the number of columns of the design matrix,
(one array per run). If only one array is provided when there
are several runs, it will be assumed that the same contrast is
desired for all runs
contrast_name : str, optional
name of the contrast
stat_type : {'t', 'F'}, optional
type of the contrast
output_type : str, optional
Type of the output map. Can be 'z_score', 'stat', 'p_value',
'effect_size' or 'effect_variance'
Returns
-------
output_image : Nifti1Image
The desired output image
"""
if self.labels_ is None or self.results_ is None:
raise ValueError('The model has not been fit yet')
if isinstance(contrast_def, np.ndarray):
con_vals = [contrast_def]
elif isinstance(contrast_def, (list, tuple)):
con_vals = contrast_def
for cidx, con in enumerate(contrast_def):
if not isinstance(con, np.ndarray):
raise ValueError('contrast_def at index %i is not an'
' array' % cidx)
else:
raise ValueError('contrast_def must be an array or list of arrays')
n_runs = len(self.labels_)
if len(con_vals) != n_runs:
warn('One contrast given, assuming it for all %d runs' % n_runs)
con_vals = con_vals * n_runs
if isinstance(output_type, _basestring):
if output_type not in ['z_score', 'stat', 'p_value', 'effect_size',
'effect_variance']:
raise ValueError('output_type must be one of "z_score", "stat",'
' "p_value","effect_size" or "effect_variance"')
else:
raise ValueError('output_type must be one of "z_score", "stat",'
' "p_value","effect_size" or "effect_variance"')
if self.memory is not None:
arg_ignore = ['labels', 'results']
mem_contrast = self.memory.cache(_fixed_effect_contrast,
ignore=arg_ignore)
else:
mem_contrast = _fixed_effect_contrast
contrast = mem_contrast(self.labels_, self.results_, con_vals,
stat_type)
estimate_ = getattr(contrast, output_type)()
# Prepare the returned images
output = self.masker_.inverse_transform(estimate_)
if contrast_name is None:
contrast_name = str(con_vals)
output.get_header()['descrip'] = (
'%s of contrast %s' % (output_type, contrast_name))
return output
X_task, labels = joblib.load('preload_HT_3mm')
labels = np.int32(labels)
# contrasts are IN ORDER -> shuffle!
new_inds = np.arange(0, X_task.shape[0])
np.random.shuffle(new_inds)
X_task = X_task[new_inds]
labels = labels[new_inds]
# subs = subs[new_inds]
X_task = StandardScaler().fit_transform(X_task)
# ARCHI task
AT_niis, AT_labels, AT_subs = joblib.load('preload_AT_3mm')
AT_X = nifti_masker.transform(AT_niis)
AT_X = StandardScaler().fit_transform(AT_X)
print('done :)')
##############################################################################
# define computation graph
##############################################################################
class SSEncoder(BaseEstimator):
def __init__(self, gain1, learning_rate, max_epochs=100,
l1=0.1, l2=0.1):
"""
Parameters
----------
lambda : float
Mediates between AE and LR. lambda==1 equates with LR only.
