def run_fc_evoke_corr(inputs):
data_path = '/data/backed_up/shared/ThalHi_MRI_2020/3dDeconvolve'
rois = '/data/backed_up/shared/ThalHi_MRI_2020/ROIs/'
# three elements expected in input
s = inputs[0] # subject name
subcortical_mask = inputs[1] # subocortical mask
cortex_masker = inputs[2] # cortex masker
#thalamus_mask = nib.load(rois+'Morel_2.5_mask.nii.gz')
#cortex_mask = nib.load(rois+'Schaefer400_2.5.nii.gz')
#cortex_masker = NiftiLabelsMasker(labels_img=cortex_mask, standardize=False)
subcortical_mask_size = np.sum(subcortical_mask.get_fdata()>0)
roi_size = len(np.unique(cortex_masker.labels_img.get_fdata()))-1
fcmat = np.zeros((subcortical_mask_size,roi_size))
conditions = ['IDS', 'EDS', 'Stay']
subcortical_evoke = {}
ctx_evoke = {}
for condition in conditions:
subcortical_evoke[condition] = np.zeros((9, subcortical_mask_size)) #subject by time by voxel
ctx_evoke[condition] = np.zeros((9, roi_size)) #subject by time by cortical ROI
# FC
fn = '/data/backed_up/shared/ThalHi_MRI_2020/3dDeconvolve/%s/%s_FIRmodel_errts.nii.gz' %(s,s)
functional_data = nib.load(fn)
cortex_ts = cortex_masker.fit_transform(functional_data)
subcortical_ts = masking.apply_mask(functional_data, subcortical_mask)
# remove censored timepoints
mts = np.mean(cortex_ts, axis = 1)
if any(mts==0):
del_i = np.where(mts==0)
cortex_ts = np.delete(cortex_ts, del_i, axis = 0)
subcortical_ts = np.delete(subcortical_ts, del_i, axis = 0)
# now try principal compoment regression
ts_len = cortex_ts.shape[0]
n_comps = np.amin([ts_len, roi_size, subcortical_mask_size]) // 20
pca = PCA(n_comps)
reduced_mat = pca.fit_transform(subcortical_ts) # Time X components
components = pca.components_
regrmodel = LinearRegression()
reg = regrmodel.fit(reduced_mat, cortex_ts) #cortex ts also time by ROI
#project regression betas from component
fcmat[:, :] = pca.inverse_transform(reg.coef_).T #reshape to cortex
# correlation and partial correlation
#fcmat[:, :] = generate_correlation_mat(thalamus_ts.T, cortex_ts.T) #th by ctx
#fcmat[:, :] = pcorr_subcortico_cortical_connectivity(thalamus_ts, cortex_ts)[400:, 0:400]
#Extract tha and cortical evoke
for condition in conditions:
fn = '/data/backed_up/shared/ThalHi_MRI_2020/3dDeconvolve/%s/%s_%s_FIR_MNI.nii.gz' %(s, s, condition)
fir_hrf = nib.load(fn)
subcortical_evoke[condition][:,:] = masking.apply_mask(fir_hrf, subcortical_mask) #time by voxel
ctx_evoke[condition][:,:] = cortex_masker.fit_transform(fir_hrf) #time by cortical ROI
return s, fcmat, subcortical_evoke, ctx_evoke
def load_denoising_data(ddict, cfg):
""" Loads the denoising parameters/data. Ugh, so ugly. Need to refactor sometime. """
f_base = cfg['f_base']
preproc_dir = op.join(cfg['save_dir'], 'preproc')
denoising_dir = op.join(cfg['save_dir'], 'denoising')
# Load in denoised data
if cfg['skip_noiseproc']: # load in preproc data if skipping noiseproc
ddict['denoised_func'] = np.load(
op.join(preproc_dir, f'{f_base}_desc-preproc_bold.npy'))
else:
ddict['denoised_func'] = np.load(
op.join(denoising_dir, f'{f_base}_desc-denoised_bold.npy'))
if 'fs' in cfg['space']:
ddict['mask'] = None
else:
ddict['mask'] = nib.load(
op.join(preproc_dir, f'{f_base}_desc-preproc_mask.nii.gz'))
if 'fs' in cfg['space']:
if cfg['iscift'] == 'y':
ddict['trs'] = [
load_and_split_cifti(f, cfg['atlas_file'], cfg['left_id'],
cfg['right_id'], cfg['subc_id'])[1]
for f in ddict['funcs']
]
else:
ddict['trs'] = [load_gifti(f, cfg)[1] for f in ddict['funcs']]
ddict['opt_n_comps'] = np.load(
op.join(denoising_dir, f'{f_base}_desc-opt_ncomps.npy'))
if cfg['hrf_model'] == 'kay' and cfg['signalproc_type'] == 'glmdenoise':
ddict['opt_hrf_idx'] = np.load(
op.join(denoising_dir, f'{f_base}_desc-opt_hrf.npy'))
else:
ddict['trs'] = [
nib.load(f).header['pixdim'][4] for f in ddict['funcs']
]
# For some reason, the line below takes a long time to run
ddict['opt_n_comps'] = masking.apply_mask(
op.join(denoising_dir, f'{f_base}_desc-opt_ncomps.nii.gz'),
ddict['mask'])
if cfg['hrf_model'] == 'kay' and cfg['signalproc_type'] == 'glmdenoise':
ddict['opt_hrf_idx'] = masking.apply_mask(
op.join(denoising_dir, f'{f_base}_desc-opt_hrf.nii.gz'),
ddict['mask'])
ddict['preproc_conf'] = pd.read_csv(op.join(
preproc_dir, f'{f_base}_desc-preproc_conf.tsv'),
sep='\t')
if not cfg['skip_signalproc']:
f_events = op.join(preproc_dir, f'{f_base}_desc-preproc_events.tsv')
ddict['preproc_events'] = pd.read_csv(f_events, sep='\t')
else:
ddict['preproc_events'] = None
ddict['run_idx'] = np.load(
op.join(preproc_dir, f"task-{cfg['c_task']}_run_idx.npy"))
return ddict
def mask_fmri_process(fmri_path, masks, sys_use='unix'):
### 1. Load and mask the data
fmri_path = ub_wind_path(fmri_path, system=sys_use) #change the path format wind-unix
mask_img_rh= masks[0] #right hemisphere mask
mask_img_rh = ub_wind_path(mask_img_rh, system=sys_use)
mask_img_lh= masks[1] #left hemisphere mask
mask_img_lh = ub_wind_path(mask_img_lh, system=sys_use)
#Apply the masks and concatenate
masked_data_rh = apply_mask(fmri_path, mask_img_rh)
masked_data_lh = apply_mask(fmri_path, mask_img_lh)
masked_data=np.hstack([masked_data_rh, masked_data_lh])
### 2. Filter ####and zscore
n_voxels = np.shape(masked_data)[1]
for voxel in range(0, n_voxels):
data_to_filter = masked_data[:,voxel]
#apply the filter
data_to_filter = TimeSeries(data_to_filter, sampling_interval=2.335)
F = FilterAnalyzer(data_to_filter, ub=0.15, lb=0.02)
data_filtered=F.filtered_boxcar.data
masked_data[:,voxel] = data_filtered
#Z score
masked_data[:,voxel] = np.array( stats.zscore( masked_data[:,voxel] ) ) ; ## zscore + 5 just to get + values
#append it and save the data
return masked_data
def std_brain(self, plot=True, mask=False):
"""Method to generate an brain of standard deviations from all brain volumes in the 4th dimension.
:param plot: {bool} whether the generated brains should be visualized
:param mask: {bool} whether the mask in the attribute :py:attr:`mask` should be applied to the image before.
:return: depending on the given options an averaged brain and / or standard-deviation-brain as numpy.array
"""
print("Computing std brain...")
if mask:
if self.data is not None:
vectorized = self.data
else:
vectorized = apply_mask(self.img, self.mask)
else:
vectorized = apply_mask(
self.img,
Nifti1Image(np.ones(self.img.shape[:3]), self.img.affine))
self.std = Nifti1Image(
np.std(vectorized, axis=0).reshape(self.img.shape[:3]),
self.img.affine)
if plot:
plot_img(self.std, title='Standard Deviation Brain')
del vectorized
self.std.uncache()
print("\tStd brain computed!")
def cov_brain(self, numproc=2, plot=True, mask=True):
"""Method to generate an brain of covariances between all brain volumes in the 4th dimension and their labels.
:param numproc: {int} number of parallel processes applied to calculate the covbrain.
:param plot: {bool} whether the generated brains should be visualized
:param mask: {bool} whether the mask in the attribute :py:attr:`mask` should be applied to the image before.
:return: the covariance brain as 3D Nifti1Image
"""
print("Computing covariance brain...")
if mask:
if self.data is not None:
vectorized = self.data
else:
vectorized = apply_mask(self.img, self.mask)
else:
vectorized = apply_mask(self.img, Nifti1Image(np.ones(self.img.shape[:3]), self.img.affine))
try:
p = Pool(numproc) # number of parallel processes
covval = np.array(p.map(partial(_cov_comp, self.targets), vectorized.T))
finally:
p.close()
p.join()
covval = np.nan_to_num(covval) # replace nan with 0
if mask:
self.cov = covval
else:
self.cov = Nifti1Image(covval.reshape(self.img.shape[:3]), self.img.affine)
self.cov.uncache()
if plot and not mask:
plot_img(self.cov, title="Covariance brain")
del vectorized
print("\tCovariance brain computed!")
def _generate_report(self):
"""Generate the visual report."""
from nilearn.image import threshold_img, load_img
from nilearn.masking import apply_mask, unmask
from niworkflows.viz.utils import plot_registration
NIWORKFLOWS_LOG.info("Generating visual report")
fixed_image_nii = load_img(self._fixed_image)
moving_image_nii = load_img(self._moving_image)
contour_nii = load_img(
self._contour) if self._contour is not None else None
if self._fixed_image_mask:
fixed_image_nii = unmask(
apply_mask(fixed_image_nii, self._fixed_image_mask),
self._fixed_image_mask,
)
# since the moving image is already in the fixed image space we
# should apply the same mask
moving_image_nii = unmask(
apply_mask(moving_image_nii, self._fixed_image_mask),
self._fixed_image_mask,
)
mask_nii = load_img(self._fixed_image_mask)
else:
mask_nii = threshold_img(fixed_image_nii, 1e-3)
n_cuts = 7
if not self._fixed_image_mask and contour_nii:
cuts = cuts_from_bbox(contour_nii, cuts=n_cuts)
else:
cuts = cuts_from_bbox(mask_nii, cuts=n_cuts)
# Call composer
compose_view(
plot_registration(
fixed_image_nii,
"fixed-image",
estimate_brightness=True,
cuts=cuts,
label=self._fixed_image_label,
contour=contour_nii,
compress=self.inputs.compress_report,
dismiss_affine=self._dismiss_affine,
),
plot_registration(
moving_image_nii,
"moving-image",
estimate_brightness=True,
cuts=cuts,
label=self._moving_image_label,
contour=contour_nii,
compress=self.inputs.compress_report,
dismiss_affine=self._dismiss_affine,
),
out_file=self._out_report,
)
def corr_decode(img, dataset, features=None, frequency_threshold=0.001,
meta_estimator=None, target_image='specificity_z'):
"""
Parameters
----------
img : :obj:`nibabel.Nifti1.Nifti1Image`
Input image to decode. Must have same affine/dimensions as dataset
mask.
dataset
A dataset with coordinates.
features : :obj:`list`, optional
List of features in dataset annotations to use for decoding.
Default is None, which uses all features available.
frequency_threshold : :obj:`float`, optional
Threshold to apply to dataset annotations. Values greater than or
equal to the threshold as assigned as label+, while values below
the threshold are considered label-. Default is 0.001.
meta_estimator : initialized :obj:`nimare.meta.cbma.base.CBMAEstimator`, optional
Defaults to MKDAChi2.
target_image : :obj:`str`, optional
Image from ``meta_estimator``'s results to use for decoding.
Dependent on estimator.
Returns
-------
out_df : :obj:`pandas.DataFrame`
A DataFrame with two columns: 'feature' (label) and 'r' (correlation
coefficient). There will be one row for each feature.
"""
# Check that input image is compatible with dataset
assert np.array_equal(img.affine, dataset.mask.affine)
# Load input data
input_data = apply_mask(img, dataset.mask)
if meta_estimator is None:
meta_estimator = MKDAChi2(dataset)
if features is None:
features = dataset.annotations.columns.values
out_df = pd.DataFrame(index=features, columns=['r'],
data=np.zeros(len(features)))
out_df.index.name = 'feature'
for feature in features:
# TODO: Search for !feature to get ids2, if possible. Will compare
# between label+ and label- without analyzing unlabeled studies.
ids = dataset.get(features=[feature],
frequency_threshold=frequency_threshold)
meta_estimator.fit(ids, corr='FDR')
feature_data = apply_mask(meta_estimator.results[target_image],
dataset.mask)
corr = np.corrcoef(feature_data, input_data)[0, 1]
out_df.loc[feature, 'r'] = corr
return out_df
def corr_dist_decode(img, dataset, features=None, frequency_threshold=0.001,
target_image='z'):
"""
Builds feature-specific distributions of correlations with input image
for image-based meta-analytic functional decoding.
Parameters
----------
img : :obj:`nibabel.Nifti1.Nifti1Image`
Input image to decode. Must have same affine/dimensions as dataset
mask.
dataset
A dataset with images.
features : :obj:`list`, optional
List of features in dataset annotations to use for decoding.
Default is None, which uses all features available.
frequency_threshold : :obj:`float`, optional
Threshold to apply to dataset annotations. Values greater than or
equal to the threshold as assigned as label+, while values below
the threshold are considered label-. Default is 0.001.
target_image : {'z', 'con'}, optional
Image type from database to use for decoding.
Returns
-------
out_df : :obj:`pandas.DataFrame`
DataFrame with a row for each feature used for decoding and two
columns: mean and std. Values describe the distributions of
correlation coefficients (in terms of Fisher-transformed z-values).
"""
# Check that input image is compatible with dataset
assert np.array_equal(img.affine, dataset.mask.affine)
# Load input data
input_data = apply_mask(img, dataset.mask)
if features is None:
features = dataset.annotations.columns.values
out_df = pd.DataFrame(index=features, columns=['mean', 'std'],
data=np.zeros(len(features), 2))
out_df.index.name = 'feature'
for feature in features:
test_imgs = dataset.get_images(features=[feature],
frequency_threshold=frequency_threshold,
image_types=[target_image])
feature_z_dist = np.zeros(len(test_imgs))
for i, test_img in enumerate(test_imgs):
feature_data = apply_mask(test_img, dataset.mask)
corr = np.corrcoef(feature_data, input_data)[0, 1]
feature_z_dist[i] = np.arctanh(corr) # transform to z for normality
out_df.loc[feature, 'mean'] = np.mean(feature_z_dist)
out_df.loc[feature, 'std'] = np.std(feature_z_dist)
return out_df
def distplot_imgs(imgs_dict,
mask_img,
ignore_zeros=True,
xlabel='Value',
title=None,
dst_dir=None,
filename=None,
**kwargs):
# extract data
imgs_names = []
imgs_data = []
for name, img in imgs_dict.items():
imgs_names.append(name)
try:
imgs_data.append(apply_mask(img, mask_img))
except ValueError:
img = resample_to_img(img, mask_img)
imgs_data.append(apply_mask(img, mask_img))
print('Images where resampled to mask resolution')
if ignore_zeros == True:
imgs_data = [img_data[img_data != 0] for img_data in imgs_data]
# plot data
plt.figure()
for name, img_data in zip(imgs_names, imgs_data):
sns.distplot(img_data, label=name, **kwargs)
# add plot information
plt.xlabel(xlabel)
plt.ylabel('Density')
plt.title(title)
# only plot legend if multiple images are given
if len(imgs_dict) > 1:
plt.legend()
# set figure background transparent
plt.gcf().patch.set_alpha(0.0)
plt.tight_layout()
if dst_dir:
if not filename:
raise ValueError('Please provide a filename')
dst_path = dst_dir + filename
plt.savefig(dst_path, dpi=600)
plt.show()
def mask(self):
'''
Apply all masks to all EPIs.
'''
self.epi_masked = {}
self.weights = {}
for key, resampled_mask in self.resampled_masks.items():
thresh = image.new_img_like(resampled_mask,
resampled_mask.get_data() > 0.01)
self.epi_masked[key] = masking.apply_mask(self.EPI, thresh)
self.weights[key] = masking.apply_mask(resampled_mask, thresh)
def get_data(download_data):
"""
Load data from dataset into global variables.
"""
# Load dataset
dset_file = op.join(get_test_data_path(), 'nidm_pain_dset.json')
dset = nimare.dataset.Dataset(dset_file)
dset.update_path(pytest.dset_dir)
# Ugly searching until better methods are implemented.
z_ids = [
id_ for id_ in dset.ids if dset.get_images(id_, imtype='z') is not None
]
z_files = dset.get_images(z_ids, imtype='z')
sample_sizes = dset.get_metadata(z_ids, 'sample_sizes')
sample_sizes = np.array([np.mean(n) for n in sample_sizes])
# Create reduced dataset for ibma
pytest.dset_z = dset.slice(z_ids)
# Now get the actual data for esma
z_imgs = [nib.load(f) for f in z_files]
z_data = apply_mask(z_imgs, dset.masker.mask_img)
pytest.data_z = z_data
pytest.sample_sizes_z = sample_sizes
# Ugly searching until better methods are implemented.
con_ids = [
id_ for id_ in dset.ids
if dset.get_images(id_, imtype='con') is not None
]
se_ids = [
id_ for id_ in dset.ids
if dset.get_images(id_, imtype='se') is not None
]
conse_ids = sorted(list(set(con_ids).intersection(se_ids)))
# Create reduced dataset for ibma
pytest.dset_conse = dset.slice(conse_ids)
# Now get the actual data for esma
con_files = dset.get_images(conse_ids, imtype='con')
se_files = dset.get_images(conse_ids, imtype='se')
sample_sizes = dset.get_metadata(conse_ids, 'sample_sizes')
sample_sizes = np.array([np.mean(n) for n in sample_sizes])
con_imgs = [nib.load(f) for f in con_files]
se_imgs = [nib.load(f) for f in se_files]
con_data = apply_mask(con_imgs, dset.masker.mask_img)
se_data = apply_mask(se_imgs, dset.masker.mask_img)
pytest.data_con = con_data
pytest.data_se = se_data
pytest.sample_sizes_con = sample_sizes
def _fit(self, dataset):
con_maps = apply_mask(self.inputs_['con_maps'], dataset.mask)
var_maps = apply_mask(self.inputs_['se_maps'], dataset.mask)
sample_sizes = np.array(
[np.mean(n) for n in self.inputs_['sample_sizes']])
images = mfx_glm(con_maps,
var_maps,
sample_sizes,
dataset.mask,
cdt=self.cdt,
q=self.q,
two_sided=self.two_sided)
return images
def mask(self):
'''
Apply all masks to all EPIs.
'''
for mask, mimg in self.resampled_masks.items():
for epi, eimg in self.EPIs.items():
thresh = image.new_img_like(mimg, mimg.get_data() > 0.01)
key = '{0}_{1}'.format(epi, mask)
key = key[:key.find('_resampled')]
self.epi_masked[key] = masking.apply_mask(eimg, thresh)
self.weights['{0}_{1}'.format(self.subject,
mask)] = masking.apply_mask(
mimg, thresh)
def Add_MRI(self, selectors):
if selectors == "brain":
masker = masking.compute_gray_matter_mask(
self.gm_imgs[0]
) #computes full brain mask... misnamed function
else:
Atlas = ATLAS()
masker = Atlas.Mask(selectors)
gm = masking.apply_mask(self.gm_imgs, masker)
wm = masking.apply_mask(self.wm_imgs, masker)
brain = gm + wm
self.masker = masker
self.features = np.hstack((self.features, brain))
def load_masked_random_runs(run3_data,
run4_data,
union_masks):
# TODO: make z-scoring optional
random1_data = [
zscore(np.nan_to_num(apply_mask(load_img(run3_file), mask_img=unionmask).T), axis=1, ddof=1)
for run3_file, unionmask
in zip(run3_data, union_masks)
]
random2_data = [
zscore(np.nan_to_num(apply_mask(load_img(run4_file), mask_img=unionmask).T), axis=1, ddof=1)
for run4_file, unionmask
in zip(run4_data, union_masks)]
return random1_data, random2_data
def do_mask(images,mask):
'''do_mask
Mask registered images - should already be in same space
images: nibabel.Nifti1Image list
a list of nifti1 objects [same size and shape]
mask: nibabel.Nifti1Image
a nifti1 object mask [same size and shape]
'''
# If we only have one image
if isinstance(images,nibabel.nifti1.Nifti1Image): images = [images]
# Make sure images are 3d (squeeze out extra dimension)
images = squeeze_fourth_dimension(images)
# Don't trust that mask is binary
mask_bin = mask.get_data().astype(bool).astype(int)
mask = nibabel.nifti1.Nifti1Image(mask_bin,affine=mask.get_affine(),header=mask.get_header())
# if ensure_finite is True, nans and infs get replaced by zeros
try:
masked_data = apply_mask(images, mask, dtype='f', smoothing_fwhm=None, ensure_finite=False)
return masked_data
except ValueError:
print("Reference and images affines do not match, or given mask and images, all data is masked.")
return numpy.nan
def main():
configFile = 'greenEyes.toml'
cfg = initializeGreenEyes(configFile)
runData = StructDict()
runData.cheating_probability = np.zeros((cfg.nStations, ))
runData.correct_prob = np.zeros((cfg.nStations, ))
runData.interpretation = getSubjectInterpretation(cfg)
runData.badVoxels = {}
runData.dataForClassification = {}
story_TRs = cfg.story_TR_2 - cfg.story_TR_1
SKIP = 10
all_data = np.zeros((cfg.nVox, cfg.nTR_run - SKIP)) # don't need to save
runData.story_data = np.zeros((cfg.nVox, story_TRs))
#### MAIN PROCESSING ###
## FUNCTION TO OPERATE OVER ALL SCANNING RUNS
scanNum = 9
for TRindex in np.arange(cfg.nTR_run - SKIP):
print('TRindex')
#for TRindex in np.arange(44):
TRnum = TRindex + 1 + SKIP # actual file number to look for
TRindex_story = TRindex - cfg.story_TR_1
full_nifti_name = convertToNifti(TRnum, scanNum, cfg)
registeredFileName = registerNewNiftiToMNI(cfg, full_nifti_name)
maskedData = apply_mask(registeredFileName, cfg.MASK)
all_data[:, TRindex] = maskedData
if TRindex_story >= 0: # we're at a story TR now
runData.story_data[:, TRindex_story] = maskedData
if np.any(TRindex_story == cfg.last_tr_in_station.astype(int)):
# NOW PREPROCESS AND CLASSIFY
runData = preprocessAndPredict(cfg, runData, TRindex_story)
else:
pass
def extract_from_masks(mask_files,
data_file,
masknames=None,
dataframe=False,
average=False):
#given a data file and a set of mask files, extract timeseries from the data
#for each mask. Returns a dictionary with the key corresponding to the mask
#(derived from the file name) and the value as a numpy array that's TR x voxels
output = [apply_mask(data_file, m) for m in mask_files]
if average:
output = [np.mean(x, axis=1) for x in output]
if (masknames is None) and (mask_files[0] is str):
masknames = [
op.splitext(op.splitext(op.basename(x))[0])[0] for x in mask_files
]
elif masknames is None:
masknames = ['mask%03d' % (x + 1) for x in range(len(mask_files))]
if dataframe and average:
output = dict(zip(masknames, output))
output = pd.DataFrame(output, columns=masknames)
elif dataframe:
output = [pd.DataFrame(x) for x in output]
else:
output = dict(zip(masknames, output))
return (output)
def standardscale(self, overwrite=True, fit=True, data=False):
"""Method to standard scale all pixel values to zero mean and 1 standard deviation.
:param overwrite: {bool} whether the original image in the attribute :py:attr:`img` should be overwritten.
:param fit: {bool} if ``Ture``, a new standard scaler is generated and fitted, if ``False``, the scaler in the
attribute :py:attr:`scaler` is used for scaling
:param data: {bool} whether to take the voxels in the attribute :py:attr:`data` as input.
:return: scaled voxels in :py:attr:`data` or :py:attr:`img` (if ``overwrite=True``)
"""
print("Rescaling pixel intensity range...")
if data:
vectorized = self.data
else:
vectorized = apply_mask(
self.img,
Nifti1Image(np.ones(self.img.shape[:3]), self.img.affine))
if fit: # instantiate new scaler, fit and transform
self.scaler = StandardScaler()
self.data = self.scaler.fit_transform(vectorized.T).T
else: # take trained scaler in self.scaler and transform
self.data = self.scaler.transform(vectorized.T).T
if overwrite:
self.img = unmask(
self.data,
Nifti1Image(np.ones(self.img.shape[:3]), self.img.affine))
self.data = None
self.img.uncache()
print("\tRescaled!")
def _generate_report(self):
"""Generates the visual report."""
from niworkflows.viz.utils import plot_registration
NIWORKFLOWS_LOG.info("Generating visual report")
anat = load_img(self._anat_file)
contour_nii = load_img(
self._contour) if self._contour is not None else None
if self._mask_file:
anat = unmask(apply_mask(anat, self._mask_file), self._mask_file)
mask_nii = load_img(self._mask_file)
else:
mask_nii = threshold_img(anat, 1e-3)
n_cuts = 7
if not self._mask_file and contour_nii:
cuts = cuts_from_bbox(contour_nii, cuts=n_cuts)
else:
cuts = cuts_from_bbox(mask_nii, cuts=n_cuts)
# Call composer
compose_view(
plot_registration(
anat,
"fixed-image",
estimate_brightness=True,
cuts=cuts,
contour=contour_nii,
compress=self.inputs.compress_report,
),
[],
out_file=self._out_report,
)
def rsfc(input):
file = input[0]
subcortical_mask = input[1]
cortex_masker = input[2]
functional_data = nib.load(file)
functional_data = nilearn.image.resample_to_img(functional_data, subcortical_mask)
cortex_ts = cortex_masker.fit_transform(functional_data)
subcortical_ts = masking.apply_mask(functional_data, subcortical_mask)
# remove censored timepoints
mts = np.mean(cortex_ts, axis = 1)
if any(mts==0):
del_i = np.where(mts==0)
cortex_ts = np.delete(cortex_ts, del_i, axis = 0)
subcortical_ts = np.delete(subcortical_ts, del_i, axis = 0)
# now try principal compoment regression
pca = PCA(239)
reduced_mat = pca.fit_transform(subcortical_ts) # Time X components
components = pca.components_
regrmodel = LinearRegression()
reg = regrmodel.fit(reduced_mat, cortex_ts) #cortex ts also time by ROI
#project regression betas from component
fcmat = pca.inverse_transform(reg.coef_).T
return fcmat
def _run_func_parallel(ddict, cfg, run, func, logger):
""" Parallelizes loading/hp-filtering of fMRI run. """
# Load data
if 'fs' in cfg['space']: # assume gifti
data, tr = load_gifti(func, return_tr=True)
tr /= 1000 # defined in msec
else:
# Load/mask data and extract stuff
tr = nib.load(func).header['pixdim'][4]
data = masking.apply_mask(func, ddict['mask'])
# By now, data is a 2D array (time x voxels)
data = hp_filter(data, tr, ddict, cfg)
# Add to run index
run_idx = np.ones(data.shape[0]) * run
if cfg['save_all']: # Save run-wise data as niftis for inspection
save_data(data,
cfg,
ddict,
par_dir='preproc',
run=run + 1,
nii=True,
desc='preproc',
dtype='bold',
skip_if_single_run=True)
return data, run_idx, tr
def get_images_df(file_paths,
mask,
dtype="f",
smoothing_fwhm=None,
ensure_finite=True):
return pandas.DataFrame(
apply_mask(file_paths, mask, dtype, smoothing_fwhm, ensure_finite))
def _generate_report(self):
""" Generates the visual report """
from niworkflows.viz.utils import plot_registration
NIWORKFLOWS_LOG.info('Generating visual report')
anat = load_img(self._anat_file)
contour_nii = load_img(self._contour) if self._contour is not None else None
if self._mask_file:
anat = unmask(apply_mask(anat, self._mask_file), self._mask_file)
mask_nii = load_img(self._mask_file)
else:
mask_nii = threshold_img(anat, 1e-3)
n_cuts = 7
if not self._mask_file and contour_nii:
cuts = cuts_from_bbox(contour_nii, cuts=n_cuts)
else:
cuts = cuts_from_bbox(mask_nii, cuts=n_cuts)
# Call composer
compose_view(
plot_registration(anat, 'fixed-image',
estimate_brightness=True,
cuts=cuts,
contour=contour_nii,
compress=self.inputs.compress_report),
[],
out_file=self._out_report
)
def calc_global_SNR(self):
self.imean = image.mean_img([self.i1, self.i2])
self.isub = image.math_img("(i2 - i1)", i1=self.i1, i2=self.i2)
self.mask = masking.compute_epi_mask(self.imean)
if not self.mask.get_fdata().any():
im = self.imean.get_fdata()[:,:,0]
thresh = filters.threshold_otsu(im)
binary = im > thresh
mask2 = self.imean.get_fdata().copy()
mask2[:,:,0]=binary
self.mask = image.new_img_like(self.imean, mask2)
imean = masking.apply_mask(self.imean, self.mask)
isub = masking.apply_mask(self.isub, self.mask)
self.SNR_global, self.SNR_global_std = self.calc_SNR(imean, isub)
def load_fmri(filename, maskname='', sigma=3):
"""
Reads 4D fmri data.
Smooths using 3D Gaussian filter
Applies mask to data:
- If mask not provided, calculates binary mask
returns fmri data matrix (Time x Voxels)
"""
img = nib.load(filename)
print(img.shape)
rep_time = img.header.get_zooms()[-1]
img = image.smooth_img(img, sigma)
if maskname != '':
img_mask = nib.load(maskname)
else:
print('Mask not provided. Calculating mask ...')
img_mask = masking.compute_background_mask(img)
img = masking.apply_mask(img, img_mask)
print('Mask applied!')
print('Detrending data!')
img = signal.clean(img,
detrend=True,
high_pass=0.01,
standardize=False,
t_r=rep_time)
return img
def apply_roi_mask_4d(image, mask, brain_mask_path=None, method='nilearn'):
'''
Apply mask to a 4d image and store resulting vectors in a measurements array (later input for pyrsa.dataset())
'''
# Apply brain mask
if isinstance(brain_mask_path, str):
brain_mask = nifti1.load(brain_mask_path)
image_array = apply_brain_mask(image, brain_mask)
else:
image_array = image.get_fdata()
if method == 'nilearn':
image_tmp = nifti1.Nifti1Image(image_array, mask.affine.copy())
masked = masking.apply_mask(image_tmp,
mask,
dtype='f',
smoothing_fwhm=None,
ensure_finite=True)
elif method == 'custom':
# Alternative to nilearn's implementation (allows for plotting of masked 3d image)
mask_array = mask.get_fdata()
masked = image_array[mask_array.astype(bool)]
masked = masked.T
elif method == '3d':
mask_array = mask.get_fdata()
image_masked = np.multiply(mask_array, image_array)
masked = nifti1.Nifti1Image(image_masked, mask.affine.copy())
return masked
def _perm(self, params):
iter_ijk, iter_df, weight_vec, conn = params
iter_ijk = np.squeeze(iter_ijk)
iter_df[['i', 'j', 'k']] = iter_ijk
k_est = self.kernel_estimator(iter_df, self.mask)
iter_ma_maps = k_est.transform(self.ids, **self.kernel_arguments)
iter_ma_maps = apply_mask(iter_ma_maps, self.mask)
iter_ma_maps *= weight_vec
iter_of_map = np.sum(iter_ma_maps, axis=0)
iter_max_value = np.max(iter_of_map)
iter_of_map = unmask(iter_of_map, self.mask)
vthresh_iter_of_map = iter_of_map.get_data().copy()
vthresh_iter_of_map[vthresh_iter_of_map < self.voxel_thresh] = 0
labeled_matrix = ndimage.measurements.label(vthresh_iter_of_map,
conn)[0]
clust_sizes = [
np.sum(labeled_matrix == val) for val in np.unique(labeled_matrix)
]
clust_sizes = clust_sizes[1:] # First cluster is zeros in matrix
if clust_sizes:
iter_max_cluster = np.max(clust_sizes)
else:
iter_max_cluster = 0
return iter_max_value, iter_max_cluster
def scramble_img(img, mask_img, random_state=None):
'''Shuffles the voxel values within a Niimg-like object
img: Niimg-like object
Input image where the voxels should be randomly shuffled.
mask_img: Niimg-like object
A mask which masks the data to brain space
random_state: int, RandomState instance or None, optional, default=None
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used by np.random.
Used when shuffle == True.
'''
rng = check_random_state(random_state)
# mask image to MNI152 template
img_data = apply_mask(img, mask_img).flatten()
n_voxels = img_data.shape[0]
# shuffle data
img_data_scrambled = rng.choice(img_data, n_voxels, replace=False)
# transform back to image
img_scrambled = unmask(img_data_scrambled, mask_img)
return img_scrambled
def peaks2maps_workflow(sleuth_file, output_dir=None, prefix=None, n_iters=10000):
"""peaks2maps workflow"""
LGR.info("Loading coordinates...")
dset = convert_sleuth_to_dataset(sleuth_file)
LGR.info("Reconstructing unthresholded maps...")
k = Peaks2MapsKernel(resample_to_mask=False)
imgs = k.transform(dset, return_type="image")
mask_img = resample_to_img(dset.mask, imgs[0], interpolation="nearest")
z_data = apply_mask(imgs, mask_img)
LGR.info("Estimating the null distribution...")
res = t_test(z_data, null="empirical", n_iters=n_iters)
res = MetaResult("t_test", maps=res, mask=mask_img)
if output_dir is None:
output_dir = os.path.dirname(os.path.abspath(sleuth_file))
else:
pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
if prefix is None:
base = os.path.basename(sleuth_file)
prefix, _ = os.path.splitext(base)
prefix += "_"
LGR.info("Saving output maps...")
res.save_maps(output_dir=output_dir, prefix=prefix)
def do_mask(images, mask):
'''do_mask
Mask registered images - should already be in same space
images: nibabel.Nifti1Image list
a list of nifti1 objects [same size and shape]
mask: nibabel.Nifti1Image
a nifti1 object mask [same size and shape]
'''
# If we only have one image
if isinstance(images, nibabel.nifti1.Nifti1Image): images = [images]
# Make sure images are 3d (squeeze out extra dimension)
images = squeeze_fourth_dimension(images)
# Don't trust that mask is binary
mask_bin = mask.get_data().astype(bool).astype(int)
mask = nibabel.nifti1.Nifti1Image(mask_bin,
affine=mask.get_affine(),
header=mask.get_header())
# if ensure_finite is True, nans and infs get replaced by zeros
try:
masked_data = apply_mask(images,
mask,
dtype='f',
smoothing_fwhm=None,
ensure_finite=False)
return masked_data
except ValueError:
print "Reference and images affines do not match, or given mask and images, all data is masked."
return numpy.nan
def peaks2maps_workflow(sleuth_file,
output_dir=None,
output_prefix=None,
n_iters=n_iters_default):
click.echo("Loading coordinates...")
dset = convert_sleuth_to_dataset(sleuth_file)
click.echo("Reconstructing unthresholded maps...")
k = Peaks2MapsKernel(resample_to_mask=False)
imgs = k.transform(dset, masked=False)
mask_img = resample_to_img(dset.mask, imgs[0], interpolation='nearest')
z_data = apply_mask(imgs, mask_img)
click.echo("Estimating the null distribution...")
res = rfx_glm(z_data, null='empirical', n_iters=n_iters)
res = MetaResult('rfx_glm', maps=res, mask=mask_img)
if output_dir is None:
output_dir = os.path.dirname(os.path.abspath(sleuth_file))
else:
pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
if output_prefix is None:
base = os.path.basename(sleuth_file)
output_prefix, _ = os.path.splitext(base)
output_prefix += '_'
click.echo("Saving output maps...")
res.save_maps(output_dir=output_dir, prefix=output_prefix)
def _run_interface(self, runtime):
ext = '.nii.gz' if self.inputs.compress else '.nii'
flat_mask = apply_mask(self.inputs.image, self.inputs.mask)
masked_image = unmask(flat_mask, self.inputs.mask)
self._results['masked_t2s'] = fname_presuffix(
self.inputs.image, suffix='_masked' + ext, newpath=runtime.cwd, use_ext=False)
masked_image.to_filename(self._results['masked_t2s'])
return runtime
def get_images_df(file_paths,
mask,
dtype="f",
smoothing_fwhm=None,
ensure_finite=True):
return pandas.DataFrame(apply_mask(file_paths, mask, dtype,
smoothing_fwhm,ensure_finite))
def _generate_report(self):
""" Generates the visual report """
from niworkflows.viz.utils import plot_registration
NIWORKFLOWS_LOG.info('Generating visual report')
fixed_image_nii = load_img(self._fixed_image)
moving_image_nii = load_img(self._moving_image)
contour_nii = load_img(self._contour) if self._contour is not None else None
if self._fixed_image_mask:
fixed_image_nii = unmask(apply_mask(fixed_image_nii,
self._fixed_image_mask),
self._fixed_image_mask)
# since the moving image is already in the fixed image space we
# should apply the same mask
moving_image_nii = unmask(apply_mask(moving_image_nii,
self._fixed_image_mask),
self._fixed_image_mask)
mask_nii = load_img(self._fixed_image_mask)
else:
mask_nii = threshold_img(fixed_image_nii, 1e-3)
n_cuts = 7
if not self._fixed_image_mask and contour_nii:
cuts = cuts_from_bbox(contour_nii, cuts=n_cuts)
else:
cuts = cuts_from_bbox(mask_nii, cuts=n_cuts)
# Call composer
compose_view(
plot_registration(fixed_image_nii, 'fixed-image',
estimate_brightness=True,
cuts=cuts,
label=self._fixed_image_label,
contour=contour_nii,
compress=self.inputs.compress_report),
plot_registration(moving_image_nii, 'moving-image',
estimate_brightness=True,
cuts=cuts,
label=self._moving_image_label,
contour=contour_nii,
compress=self.inputs.compress_report),
out_file=self._out_report
)
def mask_check(rest, mask): """ rest: 4D nifti-filename mask: 3D nifti-filename """ matrix = masking.apply_mask(rest, mask) matrix = matrix.T cnt_zeros = 0 for i in range(0, matrix.shape[0]): if np.count_nonzero(matrix[i, :]) == 0: cnt_zeros += 1 return cnt_zeros, matrix
def compute_TSNR(in_file, highpass=True, motion_corr=False):
""" Computes temporal signal-to-noise ratio (TSNR) for
a functional MRI file preprocessed with FMRIPREP.
Parameters
----------
in_file : str
Path to functional MRI file
highpass : bool
Whether to highpass the data (using the cosine basis
set in the *confounds.tsv file)
motion_corr : bool
Whether to include the six motion parameters from
motion correction in the confound regression step.
Returns
-------
out_file : str
Path to TSNR file
"""
base_dir = op.dirname(in_file)
base_file = op.basename(in_file)
if 'preproc' not in base_file:
raise ValueError("In_file should be a preprocessed file!")
# Mask BOLD file
mask = op.join(base_dir, base_file.split('preproc')[0] + 'brain_mask.nii.gz')
bold_masked = masking.apply_mask(in_file, mask)
# Get confounds and regress out of data
if highpass or motion_corr:
conf = op.join(base_dir, base_file.split('space')[0] + 'desc-confounds_regressors.tsv')
conf_df = pd.read_csv(conf, sep='\t')
conf_vars = []
if highpass:
conf_vars.extend([col for col in conf_df.columns if 'cosine' in col])
if motion_corr:
conf_vars.extend(['trans_x', 'trans_y', 'trans_z', 'rot_x', 'rot_y', 'rot_z'])
conf_df = conf_df.loc[:, conf_vars]
conf_values = np.c_[np.ones(conf_df.shape[0]), conf_df.values]
conf_params = np.linalg.lstsq(conf_values, bold_masked, rcond=-1)[0]
bold_masked = bold_masked - conf_values[:, 1:].dot(conf_params[1:, :])
# Compute tsnr
tsnr = bold_masked.mean(axis=0) / bold_masked.std(axis=0)
tsnr_img = masking.unmask(tsnr, mask)
out_file = in_file.replace('.nii.gz', '_TSNR.nii.gz')
tsnr_img.to_filename(out_file)
return out_file
def get_data_in_rois_method3(ROIs, subjects, contrasts, condir, localizerf, toppercentile):
""" returns, for individual subjects, the average contrasts values in ROIs masked by individual localizers,
tresholded to keep a toppertcentil voxels in each ROI. """
values = np.zeros((len(subjects), len(contrasts), len(ROIs)))
print ROIs
for isub, sub in enumerate(subjects):
conlist = [op.join(sub, condir, x) for x in contrasts]
localizer_img = nibabel.load(op.join(sub, localizerf))
for iroi, roi in enumerate(ROIs):
roi_img = nibabel.load(roi)
locmask = create_bestvoxels_mask(roi_img, localizer_img, toppercentile)
values[isub, :, iroi] = np.mean(apply_mask(conlist, locmask), axis=1)
return values
def compute_confounds(imgs, mask_img, n_confounds=5, get_randomized_svd=False,
compute_not_mask=False):
"""
"""
confounds = []
if not isinstance(imgs, collections.Iterable) or \
isinstance(imgs, _basestring):
imgs = [imgs, ]
img = _utils.check_niimg_4d(imgs[0])
shape = img.shape[:3]
affine = get_affine(img)
if isinstance(mask_img, _basestring):
mask_img = _utils.check_niimg_3d(mask_img)
if not _check_same_fov(img, mask_img):
mask_img = resample_img(
mask_img, target_shape=shape, target_affine=affine,
interpolation='nearest')
if compute_not_mask:
print("Non mask based confounds extraction")
not_mask_data = np.logical_not(mask_img.get_data().astype(np.int))
whole_brain_mask = masking.compute_multi_epi_mask(imgs)
not_mask = np.logical_and(not_mask_data, whole_brain_mask.get_data())
mask_img = new_img_like(img, not_mask.astype(np.int), affine)
for img in imgs:
print("[Confounds Extraction] {0}".format(img))
img = _utils.check_niimg_4d(img)
print("[Confounds Extraction] high ariance confounds computation]")
high_variance = high_variance_confounds(img, mask_img=mask_img,
n_confounds=n_confounds)
if compute_not_mask and get_randomized_svd:
signals = masking.apply_mask(img, mask_img)
non_constant = np.any(np.diff(signals, axis=0) != 0, axis=0)
signals = signals[:, non_constant]
signals = signal.clean(signals, detrend=True)
print("[Confounds Extraction] Randomized SVD computation")
U, s, V = randomized_svd(signals, n_components=n_confounds,
random_state=0)
if high_variance is not None:
confound_ = np.hstack((U, high_variance))
else:
confound_ = U
else:
confound_ = high_variance
confounds.append(confound_)
return confounds
def weighted_loadings_df(self, group_labels_file, subjid_pat=r'(?P<patid>[a-z]{2}_[0-9]{6})'):
""" Return a pandas.DataFrame with the loadings, subject IDs and groups.
This version of the sheet has the loadings signs changed for each group of subjects.
Multiply the largest blob of each IC spatial map (see `plot_ic_largest_blobs`) and the sign of the
average loading of each group.
Parameters
----------
group_labels_file: str
A CSV file with two columns: "subject_id" and "group".
The subject_ids must be in the paths contained in the Subject.mat
file and match the `subjid_pat` argument.
subjid_pat: regext str
A search regex pattern that returns one group element that
contains the subject id.
This will be used to search for subject_id in the file paths
contained in the Subjects.mat file.
Returns
-------
loadings_df: pandas.DataFrame
"""
# make sure file exists
if not op.exists(group_labels_file):
raise FileNotFoundError('The file {} has not been found.'.format(group_labels_file))
# make sure this object has been .fit()
self._update()
# let's first pick the simple version of the loadings
df = self.simple_loadings_df(group_labels_file, subjid_pat=subjid_pat)
# get the values of the largest blobs in the filtered IC maps
blobs = get_largest_blobs(self._icc_imgs)
# apply the blob mask to each corresponding IC map
masks = [apply_mask(ic_map, blob) for ic_map, blob in zip(self._icc_imgs, blobs)]
# calculate the avg per blob
blob_avgs = [mask.mean() for mask in masks]
# multiply the avg blob value with the group loadings
blob_signs = np.sign(blob_avgs)
n_ics = len(blob_avgs)
df[list(range(1, n_ics+1))] = df[list(range(1, n_ics+1))] * blob_signs
# put the dataframe back into one whole ungrouped dataframe
return df
def out_brain_confounds(epi_img, mask_img):
""" Return the 5 principal components of the signal outside the
brain.
"""
mask_img = check_niimg(mask_img)
mask_img = nibabel.Nifti1Image(
np.logical_not(mask_img.get_data()).astype(np.int),
mask_img.get_affine())
sigs = masking.apply_mask(epi_img, mask_img)
# Remove the constant signals
non_constant = np.any(np.diff(sigs, axis=0) != 0, axis=0)
sigs = sigs[:, non_constant]
sigs = signal.clean(sigs, detrend=True)
U, s, V = randomized_svd(sigs, 5, random_state=0)
return U
def load_data_whole(subj, model='logBSC_H200', metric='r'):
'''Loads data and returns them as an Encoding instance'''
fmri_data = np.hstack(
[joblib.load(('/home/data/scratch/mboos/prepro/'
'whole_fmri_subj_{}_split_{}.pkl').format(subj, i))
for i in xrange(10)]).astype('float32')
subj_mask_lobe = spenc_dir+'temporal_lobe_mask_brain_subj{0:02}bold.nii.gz'.format(subj)
subj_mask_brain = '/home/data/psyinf/forrest_gump/anondata/sub{0:03}'.format(subj) + '/templates/bold7Tp1/brain_mask.nii.gz'
img = unmask(fmri_data, mask_img=subj_mask_brain)
fmri_data = apply_mask(img, subj_mask_lobe)
model_preds = joblib.load(spenc_dir + ('MaThe/predictions/'
'{}_subj_{}_all.pkl').format(model, subj))
if metric in ('p', 'p_adj'):
scores_model = metric_functions['r'](model_preds, fmri_data)
else:
scores_model = metric_functions[metric](model_preds, fmri_data)
return Encoding(fmri_data, model_preds,
scores_model, metric=metric)
import glob
import numpy as np
import featurespace_fun as fsf
import matplotlib.pyplot as plt
from nilearn.masking import apply_mask
from nilearn.input_data import MultiNiftiMasker
from scipy.stats import norm
from nilearn.plotting import plot_glass_brain
from scipy.stats import ttest_1samp
import sys
from nibabel import load
mask = 'brainmask_group_template.nii.gz'
maps = [sorted(glob.glob('MaThe/maps/mni/model_depcor_{}*subj_*'.format(model))) for model in ['lda', 'speaker', 'emotions']]
valid_subjs = [mp.split('_')[-2] for mp in maps[0]]
for subj, ft_maps in zip(sorted(valid_subjs), zip(*maps)):
display = plot_glass_brain(None, plot_abs=False, threshold=0.001)
for ind_ft_map, color in zip(ft_maps, ['r', 'b', 'g']):
level_thr = np.percentile(apply_mask('MaThe/avg_maps/'+ind_ft_map.split('/')[-1], mask), 99.9)
display.add_contours(ind_ft_map, colors=[color], levels=[level_thr], alpha=0.6, linewidths=3.0)
display.savefig('contours_tvals_subj_{}.png'.format(subj))
plt.close()
from nilearn.masking import apply_mask
from nilearn.input_data import MultiNiftiMasker
from scipy.stats import norm
from statsmodels.sandbox.stats.multicomp import fdrcorrection0
from nilearn.plotting import plot_glass_brain
from nilearn.image import smooth_img
from scipy.stats import ttest_1samp
import sys
from nibabel import load
model = sys.argv[1]
mask = 'brainmask_group_template.nii.gz'
valid_subjs = [1,2,5,6,7,8,9,11,12,14,15,16,17,18,19]
scores = apply_mask(smooth_img(sorted(glob.glob('MaThe/avg_maps/model_depcor_{}_*'.format(model))), 5.0), mask_img=mask).astype('float32')
mean_scores = scores.mean(axis=0)
tscores, p_values = ttest_1samp(scores, 0, axis=0)
p_values[np.isnan(p_values)] = 1
which_ones = p_values > 0.05
mean_scores[which_ones] = 0
tscores[which_ones] = 0
display = fsf.plot_diff_avg_whole(mean_scores, 0.001)
display.savefig('mean_5mm_unthresh_depcor_model_whole_{}.svg'.format(model))
display.savefig('mean_5mm_unthresh_depcor_model_whole_{}.png'.format(model))
fsf.save_map_avg_whole(mean_scores, threshold=None, model='5mm_depcor_'+model)
display = fsf.plot_diff_avg_whole(tscores, 0.001)
display.savefig('ttest_5mm_unthresh_depcor_model_whole_{}.svg'.format(model))
display.savefig('ttest_5mm_unthresh_depcor_model_whole_{}.png'.format(model))
def spatial_maps_pairwise_similarity(imgs1, imgs2, mask_file, distance='correlation'):
""" Similarity values of each image in `imgs1` to each image in `imgs2`, both masked by `mask_file`.
These values are based on distance metrics, specified by `distance` argument.
The resulting similarity value is the complementary value of the distance,
i.e., '1 - <distance value>'.
The images in `imgs1` will be resampled to `imgs2` if their affine matrix don't match.
Parameters
----------
imgs1: list of niimg-like or 4D niimg-like
imgs2: list of niimg-like or 4D niimg-like
mask_file: niimg-like
distance: str
Valid values for `distance` are:
From scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2', 'manhattan'].
From scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev', 'correlation', 'dice', 'hamming',
'jaccard', 'kulsinski', 'mahalanobis', 'matching', 'minkowski',
'rogerstanimoto', 'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
'sqeuclidean', 'yule']
See the documentation for scipy.spatial.distance for details on these metrics.
Returns
-------
corrs: np.ndarray
A matrix of shape MxN, where M is len(imgs1) and N is len(imgs2).
It contains the similarity values.
"""
img1_ = niimg.load_img(imgs1)
img2_ = niimg.load_img(imgs2)
n_imgs1 = img1_.shape[-1]
n_imgs2 = img2_.shape[-1]
corrs = np.zeros((n_imgs1, n_imgs2), dtype=float)
mask_trnsf = niimg.resample_to_img(mask_file, niimg.index_img(img2_, 0),
interpolation='nearest',
copy=True)
for idx1, img1 in enumerate(niimg.iter_img(img1_)):
img1_resamp = niimg.resample_to_img(img1, niimg.index_img(img2_, 0), copy=True)
img1_masked = nimask.apply_mask(img1_resamp, mask_trnsf)
for idx2, img2 in enumerate(niimg.iter_img(img2_)):
img2_masked = nimask.apply_mask(img2, mask_trnsf)
dist = pairwise_distances(img1_masked.reshape(1, -1),
img2_masked.reshape(1, -1),
metric=distance)
# since this is a scalar value
dist = dist[0][0]
# since this is a distance, not a similarity value
corr = 1 - dist
# store it
corrs[idx1, idx2] = corr
return corrs
def run_qa(mr_paths,html_dir,software="FSL",voxdim=[2,2,2],outlier_sds=6,investigator="brainman",
nonzero_thresh=0.25,calculate_mean_image=True,view=True):
'''run_qa: a tool to generate an interactive qa report for statistical maps
mr_paths: a list of paths to brain statistical maps that can be read with nibabel [REQUIRED]
software: currently only freesurfer is supporte [default:FREESURFER]
voxdim: a list of x,y,z dimensions to resample data when normalizing [default [2,2,2]]
outlier_sds: the number of standard deviations from the mean to define an outlier [default:6]
investigator: the name (string) of an investigator to add to alerts summary page [defauflt:None]
nonzero_thresh: images with # of nonzero voxels in brain mask < this value will be flagged as thresholded [default:0.25]
calculate_mean_image: Default True, should be set to False for larger datasets where memory is an issue
view: view the web report in a browser at the end [default:True]
'''
# First resample to standard space
print("Resampling all data to %s using %s standard brain..." %(voxdim,software))
reference_file = get_standard_brain(software)
mask_file = get_standard_mask(software)
images_resamp, reference_resamp = resample_images_ref(mr_paths,reference_file,resample_dim=voxdim,interpolation="continuous")
mask = resample_img(mask_file, target_affine=np.diag(voxdim))
mask_bin = compute_epi_mask(mask)
mask_out = np.zeros(mask_bin.shape)
mask_out[mask_bin.get_data()==0] = 1
voxels_out = np.where(mask_out==1)
total_voxels = np.prod(mask_bin.shape)
total_voxels_in = len(np.where(mask_bin.get_data().flatten()==1)[0])
total_voxels_out = len(np.where(mask_out.flatten()==1)[0])
mask_out = nib.Nifti1Image(mask_out,affine=mask_bin.get_affine())
# We will save qa values for all in a data frame
results = pandas.DataFrame(columns=["voxels_in","voxels_out","standard_deviation_resamp","mean_resamp","variance_resamp","median_resamp","n_outliers_low_%ssd" %(outlier_sds),"n_outliers_high_%ssd" %(outlier_sds),"nonzero_percent_voxels_in_mask","threshold_flag"])
# We also need to save distributions for the summary page
all_histograms = []
image_names = []
# Set up directories
pwd = get_package_dir()
images_dir = "%s/img" %(html_dir)
make_dir(images_dir)
# Calculate a mean image for the entire set
if calculate_mean_image == True:
print("Calculating mean image...")
all_masked_data = apply_mask(images_resamp, mask_bin, dtype='f', smoothing_fwhm=None, ensure_finite=True)
mean_image = np.zeros(mask_bin.shape)
mean_image[mask_bin.get_data()==1] = np.mean(all_masked_data,axis=0)
mean_image = nib.Nifti1Image(mean_image,affine=mask_bin.get_affine())
mean_intensity = np.mean(mean_image.get_data()[mask_bin.get_data()==1])
histogram_data_mean = get_histogram_data(mean_image.get_data()[mask_bin.get_data()==1])
histogram_mean_counts = ",".join([str(x) for x in histogram_data_mean["counts"]])
nib.save(mean_image,"%s/mean.nii" %(html_dir))
make_stat_image("%s/mean.nii" %(html_dir),png_img_file="%s/mean.png" %(html_dir))
nib.save(reference_resamp,"%s/standard.nii" %(html_dir))
nib.save(mask_bin,"%s/mask.nii" %(html_dir))
nib.save(mask_out,"%s/mask_out.nii" %(html_dir))
make_anat_image("%s/mask.nii" %(html_dir),png_img_file="%s/mask.png" %(html_dir))
make_anat_image("%s/mask_out.nii" %(html_dir),png_img_file="%s/mask_out.png" %(html_dir))
unzip("%s/static/qa_report.zip" %(pwd),html_dir)
for m in range(0,len(mr_paths)):
mr = images_resamp[m]
mr_original = nib.load(mr_paths[m])
image_name = os.path.split(mr_paths[m])[1]
print("Generating qa report for %s" %(mr_paths[m]))
# Output folder generation
mr_folder = "%s/%s" %(html_dir,m)
make_dir(mr_folder)
mr_images = "%s/img" %(mr_folder)
make_dir(mr_images)
masked_in_data = mr.get_data()[mask_bin.get_data()==1]
masked_out_data = mr.get_data()[mask_out.get_data()==1]
mr_in_mask,mr_out_mask = make_in_out_mask(mask_bin=mask_bin,mr_folder=mr_folder,masked_in=masked_in_data,masked_out=masked_out_data,img_dir=mr_images)
# Glass brain, masked, and histogram data
make_stat_image("%s/masked.nii" %(mr_folder),png_img_file="%s/mr_masked.png" %(mr_images))
make_glassbrain_image("%s/masked.nii" %(mr_folder),png_img_file="%s/glassbrain.png" %(mr_images))
metrics = central_tendency(masked_in_data)
# Header metrics
mr_metrics = header_metrics(mr_original)
histogram_data_in = get_histogram_data(masked_in_data)
histogram_data_out = get_histogram_data(masked_out_data)
# Counting voxels (should eventually be in/out brain ratios)
count_in,count_out = count_voxels(masked_in=masked_in_data,masked_out=masked_out_data)
high_out,low_out = outliers(masked_in_data,n_std=outlier_sds)
# estimate thresholded or not. If the original image is the same shape as the mask, use it
if mr_original.shape == mask.shape:
threshold_flag,percent_nonzero = is_thresholded(mr_original,mask,threshold=nonzero_thresh)
else: # this will return biased high values because we have resampled with this standard!
threshold_flag,percent_nonzero = is_thresholded(mr,mask,threshold=nonzero_thresh)
# Add everything to table, prepare single page template
results.loc[m] = [count_in,count_out,metrics["std"],metrics["mean"],metrics["var"],metrics["med"],low_out,high_out,percent_nonzero,threshold_flag]
if calculate_mean_image == True:
template = get_template("qa_single_statmap_mean")
else:
template = get_template("qa_single_statmap")
# Things to fill into individual template
if m != 0: last_page = m-1;
else: last_page = len(mr_paths)-1;
if m != len(mr_paths)-1: next_page = m+1;
else: next_page = 0
histogram_in_counts = ",".join([str(x) for x in histogram_data_in["counts"]])
all_histograms.append(histogram_in_counts)
image_names.append(image_name)
histogram_out_counts = ",".join([str(x) for x in histogram_data_out["counts"]])
histogram_bins = '"%s"' % '","'.join([str(np.round(x,2)) for x in histogram_data_in["bins"]])
substitutions = {"NUMBER_IMAGES":len(mr_paths),
"IMAGE_NAME": image_name,
"NONZERO_VOXELS": "%0.3f" % (percent_nonzero),
"THRESHOLD_FLAG": "%s" % (threshold_flag),
"NONZERO_THRESH": "%s" % (nonzero_thresh),
"TOTAL_VOXELS":total_voxels,
"MEAN_SCORE":"%0.2f" % metrics["mean"],
"MEDIAN_SCORE":"%0.2f" % metrics["med"],
"VARIANCE_SCORE":"%0.2f" % metrics["var"],
"OUTLIERS_HIGH": high_out,
"OUTLIERS_LOW":low_out,
"OUTLIERS_STANDARD_DEVIATION":outlier_sds,
"STANDARD_DEVIATION_SCORE":"%0.2f" % metrics["std"],
"STATMAP_HISTOGRAM":histogram_in_counts,
"NEXT_PAGE":"../%s/%s.html" %(next_page,next_page),
"LAST_PAGE":"../%s/%s.html" %(last_page,last_page),
"OVERLAY_IMAGE":"%s/masked.nii" %(mr_folder),
"INVESTIGATOR":investigator
}
template = add_string(substitutions,template)
if calculate_mean_image == True:
template = add_string({"MEAN_IMAGE_HISTOGRAM":histogram_mean_counts},template)
save_template(template,"%s/%s.html" %(mr_folder,m))
# Individual pages done, now make summary pages, first the histograms
template = get_template("qa_histograms")
if calculate_mean_image == True:
statmap_histograms = ['<div class="span2 statbox purple" onTablet="span2" onDesktop="span2">\n<div class="boxchart">%s</div><div class="number" style="font-size:30px">%s</div><div class="title">images</div><div class="footer"></div></div>' %(histogram_mean_counts,len(mr_paths))]
else:
statmap_histograms = []
m = 0
for mean in results["mean_resamp"]:
if calculate_mean_image == True:
if mean >= mean_intensity:
statmap_histograms.append('<div class="span2 statbox blue" onTablet="span2"\n onDesktop="span2"><div class="boxchart">%s</div><div class="number" style="font-size:30px"><i class="icon-arrow-up"></i></div><div class="title">%s</div><div class="footer"><a href="%s/%s.html"> detail</a></div></div>' %(all_histograms[m],m,m,m))
else:
statmap_histograms.append('<div class="span2 statbox red" onTablet="span2"\n onDesktop="span2"><div class="boxchart">%s</div><div class="number" style="font-size:30px"><i class="icon-arrow-down"></i></div><div class="title">%s</div><div class="footer"><a href="%s/%s.html"> detail</a></div></div>' %(all_histograms[m],m,m,m))
else:
statmap_histograms.append('<div class="span2 statbox red" onTablet="span2"\n onDesktop="span2"><div class="boxchart">%s</div><div class="number" style="font-size:30px"></div><div class="title">%s</div><div class="footer"><a href="%s/%s.html"> detail</a></div></div>' %(all_histograms[m],m,m,m))
m+=1
template = add_string({"STATMAP_HISTOGRAMS":"\n".join(statmap_histograms),
"NUMBER_IMAGES":len(mr_paths),
"INVESTIGATOR":investigator},template)
save_template(template,"%s/histograms.html" %(html_dir))
# Summary table page and alerts
template_summary = get_template("qa_summary_table")
template_alerts = get_template("qa_alerts")
statmap_table = []; alerts_passing = []; alerts_outliers = []; alerts_thresh = []; count=0;
for res in results.iterrows():
# SUMMARY ITEMS ----
# If the image has too many zeros:
if res[1]["threshold_flag"] == True:
alerts_thresh.append('<div class="task high"><div class="desc"><div class="title">Thresholded Map</div><div>Image ID %s has been flagged as being thresholded! Nonzero voxels in mask: %s.</div></div><div class="time"><div class="date">%s</div></div></div>' %(count,res[1]["nonzero_percent_voxels_in_mask"],time.strftime("%c")))
# If the image has outliers or is thresholded:
total_outliers = res[1]["n_outliers_low_%ssd" %(outlier_sds)] + res[1]["n_outliers_high_%ssd" %(outlier_sds)]
flagged = (total_outliers > 0) | res[1]["threshold_flag"]
if flagged == True:
statmap_table.append('<tr><td>%s</td><td class="center">%s</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center"><a class="btn btn-danger" href="%s/%s.html"><i class="icon-flag zoom-in"></i></a></td></tr>' %(image_names[count],count,res[1]["mean_resamp"],res[1]["median_resamp"],res[1]["variance_resamp"],res[1]["standard_deviation_resamp"],res[1]["n_outliers_low_%ssd" %(outlier_sds)],res[1]["n_outliers_high_%ssd" %(outlier_sds)],count,count))
if res[1]["n_outliers_high_%ssd" %(outlier_sds)] > 0:
alerts_outliers.append('<div class="task medium"><div class="desc"><div class="title">Outlier High</div><div>Image ID %s has been flagged to have a high outlier</div></div><div class="time"><div class="date">%s</div><div></div></div></div>' %(count,time.strftime("%c")))
if res[1]["n_outliers_low_%ssd" %(outlier_sds)] > 0:
alerts_outliers.append('<div class="task medium"><div class="desc"><div class="title">Outlier Low</div><div>Image ID %s has been flagged to have a high outlier</div></div><div class="time"><div class="date">%s</div><div></div></div></div>' %(count,time.strftime("%c")))
# Image is passing!
else:
statmap_table.append('<tr><td>%s</td><td class="center">%s</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center">%0.2f</td><td class="center"><a class="btn btn-success" href="%s/%s.html"><i class="icon-check zoom-in"></i></a></td></tr>' %(image_names[count],count,res[1]["mean_resamp"],res[1]["median_resamp"],res[1]["variance_resamp"],res[1]["standard_deviation_resamp"],res[1]["n_outliers_low_%ssd" %(outlier_sds)],res[1]["n_outliers_high_%ssd" %(outlier_sds)],count,count))
alerts_passing.append('<div class="task low"><div class="desc"><div class="title">%s</div><div>This map has no flags as determined by the standards of only this report.</div></div><div class="time"><div class="date">%s</div><div></div></div></div>' %(image_names[count],time.strftime("%c")))
count+=1
# ALERTS ITEMS ----
# In the case of zero of any of the above
if len(alerts_thresh) == 0:
alerts_thresh = ['<div class="task high last"><div class="desc"><div class="title">No Thresholded Maps</div><div>No images have been flagged as thresholded [percent nonzero voxels in mask <= %s]</div></div><div class="time"><div class="date">%s</div></div></div>' %(nonzero_thresh,time.strftime("%c"))]
number_thresh = 0
else:
alerts_thresh[-1] = alerts_thresh[-1].replace("task high","task high last")
number_thresh = len(alerts_thresh)
if len(alerts_outliers) == 0:
alerts_outliers = ['<div class="task medium last"><div class="desc"><div class="title">No Outliers</div><div>No images have been flagged for outliers %s standard deviations in either direction.</div></div><div class="time"><div class="date">%s</div></div></div>' %(outlier_sds,time.strftime("%c"))]
number_outliers = 0
else:
alerts_outliers[-1] = alerts_outliers[-1].replace("task medium","task medium last")
number_outliers = len(alerts_outliers)
if len(alerts_passing) == 0:
alerts_passing = ['<div class="task low last"><div class="desc"><div class="title">No Passing!</div><div>No images are passing! What did you do?!</div></div><div class="time"><div class="date">%s</div></div></div>' %(time.strftime("%c"))]
# Alerts and summary template
template_alerts = add_string({"ALERTS_PASSING":"\n".join(alerts_passing),
"ALERTS_OUTLIERS":"\n".join(alerts_outliers),
"NUMBER_IMAGES":len(mr_paths),
"OUTLIERS_STANDARD_DEVIATIONS":outlier_sds,
"ALERTS_THRESH":"\n".join(alerts_thresh),
"INVESTIGATOR":investigator},template_alerts)
template_summary = add_string({"STATMAP_TABLE":"\n".join(statmap_table),
"NUMBER_IMAGES":len(mr_paths),
"OUTLIERS_STANDARD_DEVIATIONS":outlier_sds,
"INVESTIGATOR":investigator},template_summary)
save_template(template_summary,"%s/summary.html" %(html_dir))
save_template(template_alerts,"%s/alerts.html" %(html_dir))
# Finally, save the index
index_template = get_template("qa_index")
image_gallery = ['<div id="image-%s" class="masonry-thumb"><a style="background:url(%s/img/glassbrain.png) width=200px" title="%s" href="%s/%s.html"><img class="grayscale" src="%s/img/glassbrain.png" alt="%s"></a></div>' %(m,m,image_names[m],m,m,m,image_names[m]) for m in range(0,len(mr_paths)) ]
substitutions = {"GLASSBRAIN_GALLERY":"\n".join(image_gallery),
"NUMBER_OUTLIERS":number_outliers,
"NUMBER_THRESH":number_thresh,
"NUMBER_IMAGES":len(mr_paths),
"INVESTIGATOR":investigator
}
index_template = add_string(substitutions,index_template)
if calculate_mean_image == True:
index_template = add_string({"MEAN_IMAGE_HISTOGRAM":histogram_mean_counts},index_template)
save_template(index_template,"%s/index.html" %(html_dir))
# Save results to file
results.to_csv("%s/allMetrics.tsv" %(html_dir),sep="\t")
if view==True:
os.chdir(html_dir)
run_webserver(PORT=8091)
# Image 1 data
for p in range(0,len(nonzero_data_pos[0])):
datapos.loc[dfindex] = [nonzero_data_pos[0][p]+yindex,nonzero_data_pos[1][p],colorpos]
dataneg.loc[dfindex] = [nonzero_data_neg[0][p]+yindex,nonzero_data_neg[1][p],colorneg]
dfindex = dfindex+1
# We are at a new row
yindex = yindex + xdim + vertical_space
alldata = datapos.append(dataneg)
alldata.to_csv("data/thresh_data.tsv",sep="\t",index=False)
holder.to_csv("data/thresh_data.tsv",sep="\t",index=False)
# Apply mask to images, get vectors of Z score data
vectors = apply_mask(images_resamp,mask)
# Get atlas vectors
atlas_nii = nibabel.load(atlas.file)
atlas_vector = apply_mask([atlas_nii],mask)[0]
atlas_labels = ['%s' %(atlas.labels[str(int(x))].label) for x in atlas_vector]
atlas_colors = ['%s' %(atlas.color_lookup[x.replace('"',"")]) for x in atlas_labels]
# Now apply the regional mask
data_table = scatterplot_compare_vector(image_vector1=vectors[0],
image_vector2=vectors[1],
image_names=["image1","image2"],
atlas_vector=atlas_vector,
atlas_labels=atlas_labels,
atlas_colors=atlas_colors)
# Extracting a brain mask ###################################################
# Simple computation of a mask from the fMRI data
from nilearn.masking import compute_epi_mask
mask_img = compute_epi_mask(nifti_img)
mask_data = mask_img.get_data().astype(bool)
# We create a new figure
pl.figure(figsize=(3, 4))
# A plot the axial view of the mask to compare with the axial
# view of the raw data displayed previously
pl.axis('off')
pl.imshow(np.rot90(mask_data[:, :, 32]), interpolation='nearest')
pl.subplots_adjust(left=.02, bottom=.02, right=.98, top=.95)
# Applying the mask #########################################################
from nilearn.masking import apply_mask
masked_data = apply_mask(nifti_img, mask_img)
# masked_data shape is (instant number, voxel number). We can plot the first 10
# lines: they correspond to timeseries of 10 voxels on the side of the
# brain
pl.figure(figsize=(7, 5))
pl.plot(masked_data[:10].T)
pl.xlabel('Time', fontsize=16)
pl.ylabel('Voxel', fontsize=16)
pl.xlim(0, 22200)
pl.subplots_adjust(bottom=.12, top=.95, right=.95, left=.12)
pl.show()
import numpy as np
import featurespace_fun as fsf
import matplotlib.pyplot as plt
from nilearn.masking import apply_mask
from scipy.stats import norm
from statsmodels.sandbox.stats.multicomp import fdrcorrection0
from scipy.stats import ttest_1samp
import sys
models = sys.argv[1:]
#models = ['logBSC_H200_ds_conv', 'logMFS_ds']
mask = 'temporal_lobe_mask_grp_7T_test.nii.gz'
scores_bsc = np.arctanh(apply_mask(sorted(glob.glob('MaThe/avg_maps/model_{}_subj_*'.format(models[0]))), mask_img=mask))
scores_mfs = np.arctanh(apply_mask(sorted(glob.glob('MaThe/avg_maps/model_{}_subj_*'.format(models[1]))), mask_img=mask))
scores_bsc[scores_bsc<0] = 0
scores_mfs[scores_mfs<0] = 0
diff_scores = scores_bsc - scores_mfs
mean_diff = diff_scores.mean(axis=0)
tscores, p_values = ttest_1samp(diff_scores, 0, axis=0)
p_values[np.isnan(p_values)] = 1
corr_p_values = fdrcorrection0(p_values, alpha=0.05)
if not np.any(corr_p_values[0]):
print('no significant values')
sys.exit()
threshold = np.min(np.abs(tscores[p_values<0.05]))
threshold_mean = np.min(np.abs(mean_diff[p_values<0.05]))
display = fsf.plot_diff_avg(mean_diff, threshold_mean)
display.savefig('mean_diff_model_{}.svg'.format('_'.join(models)))
from scipy.stats import norm, ttest_1samp
from statsmodels.sandbox.stats.multicomp import fdrcorrection0
models = ['logBSC_H200', 'logMFS']
mask = 'temporal_lobe_mask_grp_7T_test.nii.gz'
threshold = 0.001
for model in models:
# scores = np.arctanh(apply_mask(glob.glob('MaThe/avg_maps/model_{}_p_adj_subj_*'.format(model)), mask_img=mask)).mean(axis=0)
# mean_scores = scores.mean(axis=0)
# t_values, p_values = ttest_1samp(scores, 0, axis=0)
# corr_p_values = fdrcorrection0(p_values, alpha=0.05)
## threshold = np.min(mean_scores[corr_p_values<0.05])
# threshold = 0.001
# fsf.save_map_avg('avg_unthresh', mean_scores, threshold=threshold, model=model)
# mean_scores[corr_p_values>=0.05] = 0
# display = fsf.plot_avg(mean_scores, threshold)
# fsf.save_map_avg('avg', mean_scores, threshold=threshold, model=model)
# display.savefig('mean_scores_model_{}.svg'.format(model))
# display.savefig('mean_scores_model_{}.png'.format(model))
for pc in xrange(1, 4):
scores = np.arctanh(apply_mask(glob.glob('MaThe/avg_maps/model_{}_p_adj_pc_{}_subj_*'.format(model, pc)), mask_img=mask))
mean_scores = scores.mean(axis=0)
t_values, p_values = ttest_1samp(scores, 0, axis=0)
corr_p_values = fdrcorrection0(p_values, alpha=0.05)
mean_scores[corr_p_values>=0.05] = 0
display = fsf.plot_avg(mean_scores, threshold, vmax=0.27)
display.savefig('mean_scores_model_{}_pc_{}.svg'.format(model, pc))
display.savefig('mean_scores_model_{}_pc_{}.png'.format(model, pc))
fsf.save_map_avg('avg', mean_scores, threshold=threshold, model=model+'_pc_'+str(pc))
import numpy as np
import featurespace_fun as fsf
import matplotlib.pyplot as plt
from nilearn.masking import apply_mask
from nilearn.image import smooth_img
from scipy.stats import norm
from statsmodels.sandbox.stats.multicomp import fdrcorrection0
from scipy.stats import ttest_1samp
import sys
from scipy.stats.mstats import trimmed_mean_ci
from scipy.stats import ttest_1samp, trim_mean
models = sys.argv[1:]
#models = ['logBSC_H200_ds_conv', 'logMFS_ds']
mask = 'brainmask_group_template.nii.gz'
scores_bsc = np.arctanh(apply_mask(smooth_img(glob.glob('MaThe/avg_maps/model_{}_*whole*'.format(models[0])), fwhm=3.0), mask_img=mask))
scores_mfs = np.arctanh(apply_mask(smooth_img(glob.glob('MaThe/avg_maps/model_{}_*whole*'.format(models[1])), fwhm=3.0), mask_img=mask))
diff_scores = scores_bsc - scores_mfs
mean_diff = trim_mean(diff_scores, 0.08, axis=0)
trim_mean_ci = trimmed_mean_ci(diff_scores, (0.08, 0.08), axis=0)
which_ones = np.logical_not(np.logical_or(trim_mean_ci[0,:] > 0, trim_mean_ci[1,:] < 0))
mean_diff[which_ones] = 0
display = fsf.plot_diff_avg_whole(mean_diff, 0.001)
display.savefig('mean_diff_smoothed_trim_model_{}.svg'.format('_'.join(models)))
display.savefig('mean_diff_smoothed_trim_model_{}.png'.format('_'.join(models)))
fsf.save_map_avg_whole(mean_diff, threshold=None, model='diff_smooth_trim_'+'_'.join(models))
# Simple computation of a mask from the fMRI data
from nilearn.masking import compute_epi_mask
mask_img = compute_epi_mask(haxby_files.func[0])
mask_data = mask_img.get_data().astype(bool)
# We create a new figure
plt.figure(figsize=(3, 4))
# A plot the axial view of the mask to compare with the axial
# view of the raw data displayed previously
plt.axis('off')
plt.imshow(np.rot90(mask_data[:, :, 32]), interpolation='nearest')
plt.subplots_adjust(left=.02, bottom=.02, right=.98, top=.95)
### Applying the mask #########################################################
from nilearn.masking import apply_mask
masked_data = apply_mask(haxby_files.func[0], mask_img)
# masked_data shape is (instant number, voxel number). We can plot the first 10
# lines: they correspond to timeseries of 10 voxels on the side of the
# brain
plt.figure(figsize=(7, 5))
plt.plot(masked_data[:10].T)
plt.xlabel('Time', fontsize=16)
plt.ylabel('Voxel', fontsize=16)
plt.xlim(0, 22200)
plt.subplots_adjust(bottom=.12, top=.95, right=.95, left=.12)
plt.show()
def test_apply_mask():
""" Test smoothing of timeseries extraction
"""
# A delta in 3D
# Standard masking
data = np.zeros((40, 40, 40, 2))
data[20, 20, 20] = 1
mask = np.ones((40, 40, 40))
full_mask = np.zeros((40, 40, 40))
for create_files in (False, True):
for affine in (np.eye(4), np.diag((1, 1, -1, 1)),
np.diag((.5, 1, .5, 1))):
data_img = Nifti1Image(data, affine)
mask_img = Nifti1Image(mask, affine)
with write_tmp_imgs(data_img, mask_img, create_files=create_files)\
as filenames:
series = masking.apply_mask(filenames[0], filenames[1],
smoothing_fwhm=9)
series = np.reshape(series[0, :], (40, 40, 40))
vmax = series.max()
# We are expecting a full-width at half maximum of
# 9mm/voxel_size:
above_half_max = series > .5 * vmax
for axis in (0, 1, 2):
proj = np.any(np.any(np.rollaxis(above_half_max,
axis=axis), axis=-1), axis=-1)
np.testing.assert_equal(proj.sum(),
9 / np.abs(affine[axis, axis]))
# Check that NaNs in the data do not propagate
data[10, 10, 10] = np.NaN
data_img = Nifti1Image(data, affine)
mask_img = Nifti1Image(mask, affine)
full_mask_img = Nifti1Image(full_mask, affine)
series = masking.apply_mask(data_img, mask_img, smoothing_fwhm=9)
assert_true(np.all(np.isfinite(series)))
# veriy that 4D masks are rejected
mask_img_4d = Nifti1Image(np.ones((40, 40, 40, 2)), np.eye(4))
assert_raises_regex(DimensionError, "Data must be a 3D",
masking.apply_mask, data_img, mask_img_4d)
# Check that 3D data is accepted
data_3d = Nifti1Image(np.arange(27).reshape((3, 3, 3)), np.eye(4))
mask_data_3d = np.zeros((3, 3, 3))
mask_data_3d[1, 1, 0] = True
mask_data_3d[0, 1, 0] = True
mask_data_3d[0, 1, 1] = True
data_3d = masking.apply_mask(data_3d, Nifti1Image(mask_data_3d, np.eye(4)))
assert_equal(sorted(data_3d.tolist()), [3., 4., 12.])
# Check data shape and affine
assert_raises_regex(DimensionError, "Data must be a 3D",
masking.apply_mask, data_img,
Nifti1Image(mask[20, ...], affine))
assert_raises(ValueError, masking.apply_mask,
data_img, Nifti1Image(mask, affine / 2.))
# Check that full masking raises error
assert_raises(ValueError, masking.apply_mask,
data_img, full_mask_img)
# Check weird values in data
mask[10, 10, 10] = 2
assert_raises(ValueError, masking.apply_mask,
data_img, Nifti1Image(mask, affine))
mask[15, 15, 15] = 3
assert_raises(ValueError, masking.apply_mask,
Nifti1Image(data, affine), mask_img)
cca_pearson.append(np.nan)
svi_pearson.append(np.nan)
cca_spearman.append(np.nan)
svi_spearman.append(np.nan)
sizes.loc[idx] = [0,0]
nanlog_cca.append("nan_mrthresh_empty")
nanlog_svi.append("nan_mrthresh_empty")
size_ids.append("%s_%s" %(thresh,absolute_value))
else:
# Generate a union (svi) and intersection (cca) mask
ccamask = IT.get_pairwise_deletion_mask(mrA,mrthresh,brain_mask)
svimask = IT.get_pairwise_inclusion_mask(mrA,mrthresh,brain_mask)
# COMPLETE CASE ANALYSIS (OLD PAIRWISE DELETION, intersection)
# Calculate correlation if there is overlap
if len(np.unique(ccamask.get_data())) == 2:
datacca = apply_mask([mrA,mrthresh],ccamask)
# We need at least 3 values
if np.shape(datacca)[1] > 2:
cca_pearson.append(pearsonr(datacca[0],datacca[1])[0])
cca_spearman.append(spearmanr(datacca[0],datacca[1])[0])
nanlog_cca.append("success")
else:
cca_pearson.append(np.nan)
cca_spearman.append(np.nan)
nanlog_cca.append("nan_fewer_3_values")
# Otherwise (no overlap) it is nan
else:
cca_pearson.append(np.nan)
cca_spearman.append(np.nan)
nanlog_cca.append("nan_no_overlap")
# SINGLE VALUE IMPUTATION (old pairwise inclusion, union)
def remove_mean(**kwargs):
default_conf = {
'subject': ['110929angque','110929anngio','111004edolat'],
'experiment':['memory', 'decision'],
'level': [1,3,5],
'ds_num' : [1,2,3,4,5] ,
'ds_type':['BETA'],
'file_pattern': \
"%s_%s_%s_MVPA_evidence_%s_balance_ds_%s_radius_3_searchlight_total_map.nii.gz",
'dir_pattern': "%s_%s_%s_MVPA_evidence_%s_balance_ds_%s" ,
"path": "/home/robbis/mount/fmri/memory/0_results/sl_k_3/",
"mask_dir": "/home/robbis/mount/fmri/data7/Carlo_MDM/"
}
from nilearn.masking import apply_mask, unmask
default_conf.update(kwargs)
dir_pattern = default_conf.pop("dir_pattern")
file_pattern = default_conf.pop("file_pattern")
path = default_conf.pop("path")
mask_dir = default_conf.pop("mask_dir")
key_dict = dict(zip(default_conf.keys(), range(len(default_conf.keys()))))
conf_product = itertools.product(*default_conf.values())
for elements in conf_product:
subject = elements[key_dict['subject']]
experiment = elements[key_dict['experiment']]
level = elements[key_dict['level']]
ds_num = elements[key_dict['ds_num']]
ds_type = elements[key_dict['ds_type']]
if experiment == 'memory' and ds_num > 1:
continue
file_path = os.path.join(path,
dir_pattern % (subject,
experiment,
ds_type,
level,
str(ds_num)),
file_pattern % (subject,
experiment,
ds_type,
level,
str(ds_num)),
)
img = ni.load(file_path)
mask_path = os.path.join(mask_dir,
subject,
"RESIDUALS_MVPA",
"brain_mask.nii.gz",
)
mask_img = ni.load(mask_path)
masked_data = apply_mask(img, mask_img)
mean_ = masked_data.mean()
demeaned_data = masked_data - mean_
demeaned_img = unmask(demeaned_data, mask_img)
save_pattern = file_pattern % (subject,
experiment,
ds_type,
level,
str(ds_num)
)
save_pattern_ = save_pattern[:-7]+"_demeaned.nii.gz"
save_path = os.path.join(path,
dir_pattern % (subject,
experiment,
ds_type,
level,
str(ds_num)),
save_pattern_
)
print subject, level, experiment, ds_num, mean_
ni.save(demeaned_img, save_path)
mean_img = unmask(masked_data.mean(0), mask_img)
save_pattern_ = save_pattern[:-7]+"_mean.nii.gz"
save_path = os.path.join(path,
dir_pattern % (subject,
experiment,
ds_type,
level,
str(ds_num)),
save_pattern_
)
ni.save(mean_img, save_path)
demeaned_mean = masked_data.mean(0) - masked_data.mean(0).mean()
demeaned_mean_img = unmask(demeaned_mean, mask_img)
save_pattern_ = save_pattern[:-7]+"_mean_demenead.nii.gz"
save_path = os.path.join(path,
dir_pattern % (subject,
experiment,
ds_type,
level,
str(ds_num)),
save_pattern_
)
ni.save(demeaned_mean_img, save_path)
from nilearn.masking import apply_mask
from scipi.stats import pearsonr
standard = get_standard_mask()
from glob import glob
amaps = glob("*groupA*.nii.gz")
bmaps = glob("*groupB*.nii.gz")
# Compare all A maps vs A maps
asim = pandas.DataFrame(index=amaps,columns=amaps)
bsim = pandas.DataFrame(index=bmaps,columns=bmaps)
for i in range(0,len(amaps)):
print "Processing %s of %s" %(i,len(amaps))
for j in range(0,len(amaps)):
amapi = nibabel.load(amaps[i])
amapj = nibabel.load(amaps[j])
bmapi = nibabel.load(bmaps[i])
bmapj = nibabel.load(bmaps[j])
vectorai = apply_mask(amapi,standard)
vectoraj = apply_mask(amapj,standard)
vectorbi = apply_mask(bmapi,standard)
vectorbj = apply_mask(bmapj,standard)
asim.loc[amaps[i],amaps[j]] = pearsonr(vectorai,vectoraj)[0]
bsim.loc[bmaps[i],bmaps[j]] = pearsonr(vectorbi,vectorbj)[0]
asim.to_csv("Amaps_unthresh_pearson_similarity.tsv",sep="\t")
bsim.to_csv("Bmaps_unthresh_pearson_similarity.tsv",sep="\t")
##############################################################################
# Extracting a brain mask
# Simple computation of a mask from the fMRI data
from nilearn.masking import compute_epi_mask
mask_img = compute_epi_mask(func_filename)
# Visualize it as an ROI
from nilearn.plotting import plot_roi
plot_roi(mask_img, mean_haxby)
##############################################################################
# Applying the mask to extract the corresponding time series
from nilearn.masking import apply_mask
masked_data = apply_mask(func_filename, mask_img)
# masked_data shape is (timepoints, voxels). We can plot the first 150
# timepoints from two voxels
# And now plot a few of these
import matplotlib.pyplot as plt
plt.figure(figsize=(7, 5))
plt.plot(masked_data[:150, :2])
plt.xlabel('Time [TRs]', fontsize=16)
plt.ylabel('Intensity', fontsize=16)
plt.xlim(0, 150)
plt.subplots_adjust(bottom=.12, top=.95, right=.95, left=.12)
show()
def create_bestvoxels_mask(roi_img, localizer_img, toppercentile=25):
""" select voxels within roi_img having the largest values in localizer_img """
masked_data = apply_mask(localizer_img, roi_img)
threshold = scoreatpercentile(masked_data, 100 - toppercentile)
mask = binarize_img(localizer_img, threshold)
return intersect_masks((roi_img, mask), threshold=1)
from nilearn.input_data import MultiNiftiMasker
from scipy.stats import norm
from statsmodels.sandbox.stats.multicomp import fdrcorrection0
from nilearn.plotting import plot_glass_brain
from scipy.stats import ttest_1samp
import sys
from nibabel import load
from nilearn.image import smooth_img
models = sys.argv[1:]
mask = 'brainmask_group_template.nii.gz'
valid_subjs = [1,2,5,6,7,8,9,11,12,14,15,16,17,18,19]
scores_bsc = np.arctanh(apply_mask(smooth_img(sorted(glob.glob('MaThe/avg_maps/model_{}_*whole*'.format(models[0]))), 3.0), mask_img=mask)).astype('float32')
scores_mfs = np.arctanh(apply_mask(smooth_img(sorted(glob.glob('MaThe/avg_maps/model_{}_*whole*'.format(models[1]))), 3.0), mask_img=mask)).astype('float32')
scores_bsc[scores_bsc<0] = 0
scores_mfs[scores_mfs<0] = 0
diff_scores = scores_bsc - scores_mfs
mean_diff = diff_scores.mean(axis=0)
tscores, p_values = ttest_1samp(diff_scores, 0, axis=0)
p_values[np.isnan(p_values)] = 1
which_ones = p_values > 0.05
mean_diff[which_ones] = 0
tscores[which_ones] = 0
display = fsf.plot_diff_avg_whole(mean_diff, 0.001)
display.savefig('mean_unthresh_diff_model_whole_3mm_{}.svg'.format('_'.join(models)))
display.savefig('mean_unthresh_diff_model_whole_3mm_{}.png'.format('_'.join(models)))
fsf.save_map_avg_whole(mean_diff, threshold=None, model='3mm_diff_'+'_'.join(models))
display = fsf.plot_diff_avg_whole(tscores, 0.001)
# Now we will count the number of voxels and compare the two methods
grf_files = glob("%s/*.nii.gz" %output_folder)
grf_files.sort()
input_files.sort()
voxel_counts = pandas.DataFrame(columns=["rft","thresh_0.0","thresh_1.0","thresh_2.0"])
brain_mask = nibabel.load(mask)
brain_voxels = len(brain_mask.get_data()[brain_mask.get_data()!=0])
for f in range(0,len(grf_files)):
f1 = IT.t_to_z(nibabel.load(input_files[f]),46)
f2 = nibabel.load(grf_files[f])
print "Comparing %s vs %s" %(os.path.basename(grf_files[f]),os.path.basename(input_files[f]))
mrthresh = IT.threshold_abs(f1,thresholds=[0.0,1.0,2.0])
masked_mrthresh = [(thresh,apply_mask([nii],brain_mask)) for thresh,nii in mrthresh.iteritems()]
# Count voxels in each
image_name = os.path.basename(input_files[f])
voxel_counts.loc[image_name] = (len(f2.get_data()[f2.get_data()!=0]),
len(masked_mrthresh[0][1][0][masked_mrthresh[0][1][0]!=0]),
len(masked_mrthresh[1][1][0][masked_mrthresh[1][1][0]!=0]),
len(masked_mrthresh[2][1][0][masked_mrthresh[2][1][0]!=0]))
voxel_counts["absolute_difference_0.0"] = voxel_counts["thresh_0.0"] - voxel_counts["rft"]
voxel_counts["absolute_difference_1.0"] = voxel_counts["thresh_1.0"] - voxel_counts["rft"]
voxel_counts["absolute_difference_2.0"] = voxel_counts["thresh_2.0"] - voxel_counts["rft"]
voxel_counts["perc_brainvox_0.0"] = voxel_counts["thresh_0.0"] / brain_voxels
voxel_counts["perc_brainvox_1.0"] = voxel_counts["thresh_1.0"] / brain_voxels
voxel_counts["perc_brainvox_2.0"] = voxel_counts["thresh_2.0"] / brain_voxels
voxel_counts["perc_brainvox_rft"] = voxel_counts["rft"] / brain_voxels
voxel_counts["perc_difference_0.0"] = voxel_counts["absolute_difference_0.0"] / brain_voxels