def _mask_and_reduce_single(masker,
img,
confound,
reduction_ratio=None,
reduction_method=None,
n_samples=None,
memory=None,
memory_level=0,
random_state=None):
"""Utility function for multiprocessing from MaskReducer"""
this_data = masker.transform(img, confound)
# Now get rid of the img as fast as possible, to free a
# reference count on it, and possibly free the corresponding
# data
del img
random_state = check_random_state(random_state)
data_n_samples, data_n_features = this_data.shape
if reduction_ratio is None:
assert n_samples is not None
n_samples = min(n_samples, data_n_samples)
else:
n_samples = int(ceil(data_n_samples * reduction_ratio))
if reduction_method == 'svd':
if n_samples <= data_n_features // 4:
U, S, _ = cache(randomized_svd,
memory,
memory_level=memory_level,
func_memory_level=3)(this_data.T,
n_samples,
transpose=True,
random_state=random_state)
U = U.T
else:
U, S, _ = cache(linalg.svd,
memory,
memory_level=memory_level,
func_memory_level=3)(this_data.T,
full_matrices=False)
U = U.T[:n_samples].copy()
S = S[:n_samples]
U = U * S[:, np.newaxis]
elif reduction_method == 'rf':
Q = cache(randomized_range_finder,
memory,
memory_level=memory_level,
func_memory_level=3)(this_data,
n_samples,
n_iter=3,
random_state=random_state)
U = Q.T.dot(this_data)
elif reduction_method == 'ss':
indices = np.floor(np.linspace(0, this_data.shape[0] - 1,
n_samples)).astype('int')
U = this_data[indices]
else:
U = this_data
return U
def test_cache_shelving():
with tempfile.TemporaryDirectory() as temp_dir:
joblib_dir = Path(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f')
mem = Memory(cachedir=temp_dir, verbose=0)
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(_get_subdirs(joblib_dir)), 1)
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(_get_subdirs(joblib_dir)), 1)
def _mask_and_reduce_single(masker,
img, confound,
reduction_ratio=None,
reduction_method=None,
n_samples=None,
memory=None,
memory_level=0,
random_state=None):
"""Utility function for multiprocessing from MaskReducer"""
this_data = masker.transform(img, confound)
# Now get rid of the img as fast as possible, to free a
# reference count on it, and possibly free the corresponding
# data
del img
random_state = check_random_state(random_state)
data_n_samples, data_n_features = this_data.shape
if reduction_ratio is None:
assert n_samples is not None
n_samples = min(n_samples, data_n_samples)
else:
n_samples = int(ceil(data_n_samples * reduction_ratio))
if reduction_method == 'svd':
if n_samples <= data_n_features // 4:
U, S, _ = cache(randomized_svd, memory,
memory_level=memory_level,
func_memory_level=3)(this_data.T,
n_samples,
transpose=True,
random_state=random_state)
U = U.T
else:
U, S, _ = cache(linalg.svd, memory,
memory_level=memory_level,
func_memory_level=3)(this_data.T,
full_matrices=False)
U = U.T[:n_samples].copy()
S = S[:n_samples]
U = U * S[:, np.newaxis]
elif reduction_method == 'rf':
Q = cache(randomized_range_finder, memory,
memory_level=memory_level,
func_memory_level=3)(this_data,
n_samples, n_iter=3,
random_state=random_state)
U = Q.T.dot(this_data)
elif reduction_method == 'ss':
indices = np.floor(np.linspace(0, this_data.shape[0] - 1,
n_samples)).astype('int')
U = this_data[indices]
else:
U = this_data
return U
def test_cache_shelving():
try:
temp_dir = tempfile.mkdtemp()
job_glob = os.path.join(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f', '*')
mem = Memory(cachedir=temp_dir, verbose=0)
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(glob.glob(job_glob)), 1)
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(glob.glob(job_glob)), 1)
finally:
del mem
shutil.rmtree(temp_dir, ignore_errors=True)
def cached_mask_and_reduce_single(masker,
img,
confound,
reduction_ratio=1,
reduction_method=None,
n_samples=None,
memory_level=0,
memory=Memory(cachedir=None),
random_state=0,
as_shelved_list=False):
if as_shelved_list:
return cache(_mask_and_reduce_single,
memory=memory,
memory_level=memory_level,
func_memory_level=0).call_and_shelve(
masker,
img,
confound,
reduction_ratio=reduction_ratio,
reduction_method=reduction_method,
n_samples=n_samples,
memory=memory,
memory_level=memory_level,
random_state=random_state)
else:
return _mask_and_reduce_single(masker,
img,
confound,
reduction_ratio=reduction_ratio,
reduction_method=reduction_method,
n_samples=n_samples,
memory=memory,
memory_level=memory_level,
random_state=random_state)
def _region_extractor_cache(maps_img, mask_img=None, min_region_size=2500,
threshold=1., thresholding_strategy='ratio_n_voxels',
extractor='local_regions',
memory=Memory(cachedir=None), memory_level=0):
"""Region Extraction with caching built upon helper functions
See nilearn.regions.RegionExtractor for documentation and related
"""
if thresholding_strategy == 'ratio_n_voxels':
print("[Thresholding] using maps ratio with threshold={0}"
" ".format(threshold))
threshold_maps = _threshold_maps_ratio(maps_img, threshold)
else:
if thresholding_strategy == 'percentile':
threshold = "{0}%".format(threshold)
print("[Thresholding] using threshold_img with threshold={0}"
" ".format(threshold))
threshold_maps = threshold_img(maps_img, mask_img=mask_img,
threshold=threshold)
print("Done thresholding")
print("[Region Extraction] with {0}".format(extractor))
regions_img, _ = cache(connected_regions, memory,
memory_level=memory_level,
func_memory_level=1)(threshold_maps,
min_region_size=min_region_size,
extract_type=extractor)
return regions_img
def cached_mask_and_reduce_single(masker, img, confound,
reduction_ratio=1,
reduction_method=None,
n_samples=None,
memory_level=0,
memory=Memory(cachedir=None),
random_state=0,
as_shelved_list=False):
if as_shelved_list:
return cache(
_mask_and_reduce_single, memory=memory,
memory_level=memory_level,
func_memory_level=0).call_and_shelve(
masker, img,
confound,
reduction_ratio=reduction_ratio,
reduction_method=reduction_method,
n_samples=n_samples,
memory=memory,
memory_level=memory_level,
random_state=random_state)
else:
return _mask_and_reduce_single(
masker, img, confound,
reduction_ratio=reduction_ratio,
reduction_method=reduction_method,
n_samples=n_samples,
memory=memory,
memory_level=memory_level,
random_state=random_state)
def test_cache_memory_level():
with tempfile.TemporaryDirectory() as temp_dir:
joblib_dir = Path(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f')
mem = Memory(cachedir=temp_dir, verbose=0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=1)(2)
assert_equal(len(_get_subdirs(joblib_dir)), 0)
cache_mixin.cache(f, Memory(cachedir=None))(2)
assert_equal(len(_get_subdirs(joblib_dir)), 0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=3)(2)
assert_equal(len(_get_subdirs(joblib_dir)), 1)
cache_mixin.cache(f, mem)(3)
assert_equal(len(_get_subdirs(joblib_dir)), 2)
def test_cache_memory_level():
temp_dir = tempfile.mkdtemp()
job_glob = os.path.join(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f', '*')
mem = Memory(cachedir=temp_dir, verbose=0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=1)(2)
assert_equal(len(glob.glob(job_glob)), 0)
cache_mixin.cache(f, Memory(cachedir=None))(2)
assert_equal(len(glob.glob(job_glob)), 0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=3)(2)
assert_equal(len(glob.glob(job_glob)), 2)
cache_mixin.cache(f, mem)(3)
assert_equal(len(glob.glob(job_glob)), 3)
def test_cache_memory_level():
temp_dir = tempfile.mkdtemp()
job_glob = os.path.join(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f', '*')
mem = Memory(cachedir=temp_dir, verbose=0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=1)(2)
assert_equal(len(glob.glob(job_glob)), 0)
cache_mixin.cache(f, Memory(cachedir=None))(2)
assert_equal(len(glob.glob(job_glob)), 0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=3)(2)
assert_equal(len(glob.glob(job_glob)), 2)
cache_mixin.cache(f, mem)(3)
assert_equal(len(glob.glob(job_glob)), 3)
def test_cache_shelving():
try:
temp_dir = tempfile.mkdtemp()
job_glob = os.path.join(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f', '*')
mem = Memory(cachedir=temp_dir, verbose=0)
if LooseVersion(sklearn.__version__) >= LooseVersion('0.15'):
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(glob.glob(job_glob)), 1)
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(glob.glob(job_glob)), 1)
else:
assert_raises_regex(ValueError, 'Shelving is only available'
' if scikit-learn >= 0.15'
' is installed.',
cache_mixin.cache,
f, mem, shelve=True)
finally:
del mem
shutil.rmtree(temp_dir, ignore_errors=True)
def test_cache_shelving():
try:
temp_dir = tempfile.mkdtemp()
job_glob = os.path.join(temp_dir, 'joblib', 'nilearn', 'tests',
'test_cache_mixin', 'f', '*')
mem = Memory(cachedir=temp_dir, verbose=0)
if LooseVersion(sklearn.__version__) >= LooseVersion('0.15'):
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(glob.glob(job_glob)), 1)
res = cache_mixin.cache(f, mem, shelve=True)(2)
assert_equal(res.get(), 2)
assert_equal(len(glob.glob(job_glob)), 1)
else:
assert_raises_regex(ValueError, 'Shelving is only available'
' if scikit-learn >= 0.15'
' is installed.',
cache_mixin.cache,
f,
mem,
shelve=True)
finally:
del mem
shutil.rmtree(temp_dir, ignore_errors=True)
def fit(self, X, y=None):
"""Fit the covariance estimator to the given time series for each
subject.
Parameters
----------
X : list of numpy.ndarray, shape for each (n_samples, n_features)
The input subjects time series.
Returns
-------
self : ConnectivityMatrix instance
The object itself. Useful for chaining operations.
"""
self.cov_estimator_ = clone(self.cov_estimator)
if not hasattr(X, "__iter__"):
raise ValueError("'subjects' input argument must be an iterable. "
"You provided {0}".format(X.__class__))
subjects_types = [type(s) for s in X]
if set(subjects_types) != set([np.ndarray]):
raise ValueError("Each subject must be 2D numpy.ndarray.\n You "
"provided {0}".format(str(subjects_types)))
subjects_dims = [s.ndim for s in X]
if set(subjects_dims) != set([2]):
raise ValueError("Each subject must be 2D numpy.ndarray.\n You"
"provided arrays of dimensions "
"{0}".format(str(subjects_dims)))
n_subjects = [s.shape[1] for s in X]
if len(set(n_subjects)) > 1:
raise ValueError("All subjects must have the same number of "
"features.\nYou provided: "
"{0}".format(str(n_subjects)))
if self.kind == 'tangent':
covariances = [self.cov_estimator_.fit(x).covariance_ for x in X]
self.mean_ = cache(_geometric_mean,
memory=self.memory,
func_memory_level=2,
memory_level=self.memory_level)(covariances,
max_iter=30,
tol=1e-7)
self.whitening_ = _map_eigenvalues(lambda x: 1. / np.sqrt(x),
self.mean_)
return self
def test_cache_memory_level():
temp_dir = tempfile.mkdtemp()
job_glob = os.path.join(temp_dir, 'joblib', '*')
mem = Memory(cachedir=temp_dir, verbose=0)
cache_mixin.cache(f, mem)(2)
assert_true(len(glob.glob(job_glob)) == 0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=1)(2)
assert_true(len(glob.glob(job_glob)) == 0)
cache_mixin.cache(f, mem, func_memory_level=2, memory_level=3)(2)
assert_true(len(glob.glob(job_glob)) == 2)
def fit(self, X, y=None):
"""Fit the covariance estimator to the given time series for each
subject.
Parameters
----------
X : list of numpy.ndarray, shape for each (n_samples, n_features)
The input subjects time series.
Returns
-------
self : ConnectivityMatrix instance
The object itself. Useful for chaining operations.
"""
self.cov_estimator_ = clone(self.cov_estimator)
if not hasattr(X, "__iter__"):
raise ValueError("'subjects' input argument must be an iterable. "
"You provided {0}".format(X.__class__))
subjects_types = [type(s) for s in X]
if set(subjects_types) != set([np.ndarray]):
raise ValueError("Each subject must be 2D numpy.ndarray.\n You "
"provided {0}".format(str(subjects_types)))
subjects_dims = [s.ndim for s in X]
if set(subjects_dims) != set([2]):
raise ValueError("Each subject must be 2D numpy.ndarray.\n You"
"provided arrays of dimensions "
"{0}".format(str(subjects_dims)))
n_subjects = [s.shape[1] for s in X]
if len(set(n_subjects)) > 1:
raise ValueError("All subjects must have the same number of "
"features.\nYou provided: "
"{0}".format(str(n_subjects)))
if self.kind == 'tangent':
covariances = [self.cov_estimator_.fit(x).covariance_ for x in X]
self.mean_ = cache(
_geometric_mean, memory=self.memory, func_memory_level=2,
memory_level=self.memory_level)(
covariances, max_iter=30, tol=1e-7)
self.whitening_ = _map_eigenvalues(lambda x: 1. / np.sqrt(x),
self.mean_)
return self
def eddy_current_correction(img, file_bvals, file_bvecs, write_dir, mem,
acqp='b0_acquisition_params_AP.txt'):
""" Perform Eddy current correction on diffusion data
Todo: do topup + eddy in one single command
"""
import nibabel as nib
bvals = np.loadtxt(file_bvals)
mean_img = get_mean_unweighted_image(nib.load(img), bvals)
mean_img.to_filename(os.path.join(write_dir, 'mean_unweighted.nii.gz'))
mask = compute_epi_mask(mean_img)
mask_file = os.path.join(write_dir, 'mask.nii.gz')
nib.save(mask, mask_file)
corrected = os.path.join(os.path.dirname(img),
'ed' + os.path.basename(img))
index = np.ones(len(bvals), np.uint8)
index_file = os.path.join(write_dir, 'index.txt')
np.savetxt(index_file, index)
cmd = 'fsl5.0-eddy_correct --acqp=%s --bvals=%s --bvecs=%s --imain=%s --index=%s --mask=%s --out=%s' % (
acqp, file_bvals, file_bvecs, img, index_file, mask_file, corrected)
cmd = 'fsl5.0-eddy_correct %s %s %d' % (img, corrected, 0)
print(cmd)
print(commands.getoutput(cmd))
print(cache(commands.getoutput, mem)(cmd))
def subject_covariance(
estimator, niimgs, mask_img, parameters,
confounds=None,
ref_memory_level=0,
memory=Memory(cachedir=None),
connectivity=None,
verbose=0,
copy=True):
data, affine = cache(
filter_and_mask, memory=memory, ref_memory_level=ref_memory_level,
memory_level=2,
ignore=['verbose', 'memory', 'ref_memory_level', 'copy'])(
niimgs, mask_img, parameters,
ref_memory_level=ref_memory_level,
memory=memory,
verbose=verbose,
confounds=confounds,
copy=copy)
estimator = clone(estimator)
if connectivity is not None:
estimator.fit(data, connectivity=connectivity)
else:
estimator.fit(data)
return estimator.covariance_
def _build_parcellations(all_subjects_data, mask, n_parcellations=100,
n_parcels=1000, n_bootstrap_samples=None,
random_state=None, memory=Memory(cachedir=None),
n_jobs=1, verbose=False):
"""Build the parcellations for the RPBI framework.
Parameters
----------
all_subjects_data : array_like, shape=(n_samples, n_voxels)
Masked subject images as an array.
mask : ndarray of booleans
Mask that has been applied on the initial images to obtain
`all_subjects_data`.
n_parcellations : int,
The number of parcellations to be built and used to extract
signal averages from the data.
n_parcels : int,
Number of parcels for the parcellations.
n_bootstrap_samples : int,
Number of subjects to be used to build the parcellations. The subjects
are randomly drawn with replacement.
If set to None, n_samples subjects are drawn, which correspond to
a bootstrap draw.
random_state : int,
Random numbers seed for reproducible results.
memory : instance of joblib.Memory or string
Used to cache the masking process.
By default, no caching is done. If a string is given, it is the
path to the caching directory.
n_jobs : int,
Number of parallel workers.
If 0 is provided, all CPUs are used.
A negative number indicates that all the CPUs except (|n_jobs| - 1) ones
will be used.
verbose : boolean,
Activate verbose mode (default is False).
Returns
-------
parcelled_data : np.ndarray, shape=(n_parcels_tot, n_subjs)
Data for all subjects after mean signal extraction with all the
parcellations that have been created.
ward_labels : np.ndarray, shape=(n_vox * n_wards, )
Voxel-to-parcel map for all the parcellations. Useful to perform
inverse transforms.
TODO
----
- Deal with NaNs in the original data (FeatureAgglomeration cannot fit
when NaNs are present in the data). Median imputation?
"""
# initialize the seed of the random generator
rng = check_random_state(random_state)
# check n_jobs (number of CPUs)
n_jobs = check_n_jobs(n_jobs)
n_samples = all_subjects_data.shape[0]
if n_bootstrap_samples is None:
n_bootstrap_samples = n_samples
# Compute connectivity
shape = mask.shape
connectivity = image.grid_to_graph(
n_x=shape[0], n_y=shape[1], n_z=shape[2], mask=mask)
# Build parcellations
draw = rng.randint(n_samples, size=n_bootstrap_samples * n_parcellations)
draw = draw.reshape((n_parcellations, -1))
ret = joblib.Parallel(n_jobs=n_jobs)(joblib.delayed(
cache(_ward_fit_transform, memory, verbose=verbose))
(all_subjects_data, draw[i], connectivity, n_parcels, i * n_parcels)
for i in range(n_parcellations))
# reduce results
parcelled_data_parts, ward_labels = zip(*ret)
parcelled_data = np.hstack((parcelled_data_parts))
ward_labels = np.ravel(ward_labels)
return parcelled_data, ward_labels
def _build_parcellations(all_subjects_data,
mask,
n_parcellations=100,
n_parcels=1000,
n_bootstrap_samples=None,
random_state=None,
memory=Memory(cachedir=None),
n_jobs=1,
verbose=False):
"""Build the parcellations for the RPBI framework.
Parameters
----------
all_subjects_data : array_like, shape=(n_samples, n_voxels)
Masked subject images as an array.
mask : ndarray of booleans
Mask that has been applied on the initial images to obtain
`all_subjects_data`.
n_parcellations : int,
The number of parcellations to be built and used to extract
signal averages from the data.
n_parcels : int,
Number of parcels for the parcellations.
n_bootstrap_samples : int,
Number of subjects to be used to build the parcellations. The subjects
are randomly drawn with replacement.
If set to None, n_samples subjects are drawn, which correspond to
a bootstrap draw.
random_state : int,
Random numbers seed for reproducible results.
memory : instance of joblib.Memory or string
Used to cache the masking process.
By default, no caching is done. If a string is given, it is the
path to the caching directory.
n_jobs : int,
Number of parallel workers.
If 0 is provided, all CPUs are used.
A negative number indicates that all the CPUs except (|n_jobs| - 1) ones
will be used.
verbose : boolean,
Activate verbose mode (default is False).
Returns
-------
parcelled_data : np.ndarray, shape=(n_parcels_tot, n_subjs)
Data for all subjects after mean signal extraction with all the
parcellations that have been created.
ward_labels : np.ndarray, shape=(n_vox * n_wards, )
Voxel-to-parcel map for all the parcellations. Useful to perform
inverse transforms.
TODO
----
- Deal with NaNs in the original data (WardAgglomeration cannot fit
when NaNs are present in the data). Median imputation?
"""
# initialize the seed of the random generator
rng = check_random_state(random_state)
# check n_jobs (number of CPUs)
n_jobs = check_n_jobs(n_jobs)
n_samples = all_subjects_data.shape[0]
if n_bootstrap_samples is None:
n_bootstrap_samples = n_samples
# Compute connectivity
shape = mask.shape
connectivity = image.grid_to_graph(n_x=shape[0],
n_y=shape[1],
n_z=shape[2],
mask=mask)
# Build parcellations
draw = rng.randint(n_samples, size=n_bootstrap_samples * n_parcellations)
draw = draw.reshape((n_parcellations, -1))
ret = joblib.Parallel(n_jobs=n_jobs)(
joblib.delayed(cache(_ward_fit_transform, memory, verbose=verbose))(
all_subjects_data, draw[i], connectivity, n_parcels, i * n_parcels)
for i in range(n_parcellations))
# reduce results
parcelled_data_parts, ward_labels = zip(*ret)
parcelled_data = np.hstack((parcelled_data_parts))
ward_labels = np.ravel(ward_labels)
return parcelled_data, np.ravel(ward_labels)