def collapse_mask(mask, auto_label=True, custom_mask=None):
"""collapse separate masks into one mask with multiple integers
overlapping areas are ignored
Args:
mask: nibabel or Brain_Data instance
custom_mask: nibabel instance or string to file path; optional
Returns:
out: Brain_Data instance of a mask with different integers indicating
different masks
"""
from nltools.data import Brain_Data
if not isinstance(mask, Brain_Data):
if isinstance(mask, nib.Nifti1Image):
mask = Brain_Data(mask, mask=custom_mask)
else:
raise ValueError("Make sure mask is a nibabel or Brain_Data "
"instance.")
if len(mask.shape()) > 1:
if len(mask) > 1:
out = mask.empty()
# Create list of masks and find any overlaps
m_list = []
for x in range(len(mask)):
m_list.append(mask[x].to_nifti())
intersect = intersect_masks(m_list, threshold=1, connected=False)
intersect = Brain_Data(
nib.Nifti1Image(np.abs(intersect.get_data() - 1),
intersect.get_affine()),
mask=custom_mask,
)
merge = []
if auto_label:
# Combine all masks into sequential order
# ignoring any areas of overlap
for i in range(len(m_list)):
merge.append(
np.multiply(
Brain_Data(m_list[i], mask=custom_mask).data,
intersect.data) * (i + 1))
out.data = np.sum(np.array(merge).T, 1).astype(int)
else:
# Collapse masks using value as label
for i in range(len(m_list)):
merge.append(
np.multiply(
Brain_Data(m_list[i], mask=custom_mask).data,
intersect.data))
out.data = np.sum(np.array(merge).T, 1)
return out
else:
warnings.warn("Doesn't need to be collapased")
def get_trialtype_pain_regressors(self,nifti_data,onset_file):
print("importing nifti")
#import the nifti
if (os.path.isfile(nifti_data + "nltoolstandard.nii.gz")):
msmrl1 = Brain_Data(
nifti_data + "nltoolstandard.nii.gz")
else:
msmrl1 = Brain_Data(
nifti_data + ".nii.gz")
msmrl1.write(nifti_data + "nltoolstandard.nii.gz")
#preprocess the nifti?
print("importing onsets")
#import the onset
onsets = onsets_to_dm(
onset_file,
TR=2,
runLength=msmrl1.shape()[0]
)
#process the onset files
#
onsets.sampling_rate=2
onsets_convolved=onsets.convolve()
for c in onsets_convolved.columns:
if sum(onsets_convolved.ix[:, c]) <= 0:
print('deleting '+ str(c))
del onsets_convolved[c]
onsets_convolved['linearterm']=range(1,361)
onsets_convolved['quadraticterm']=[pow(x,2) for x in onsets_convolved['linearterm']]
onsets_convolved['cubicterm']=[pow(x,3) for x in onsets_convolved['linearterm']]
onsets_convolved['ones']=[1]*360
msmrl1.X=onsets_convolved
print("convolved onsets; regressing...")
#regress
regression=msmrl1.regress()
print("Regressing; calculating similarity...")
msm_predicted_pain = regression['beta'].similarity(self.stats['weight_map'], 'dot_product')
onset_colnames = onsets_convolved.columns.tolist()
msm_predicted_pain_dict={}
for i, b in enumerate(msm_predicted_pain):
msm_predicted_pain_dict[onset_colnames[i]] = b
return msm_predicted_pain_dict
def test_data(tmpdir):
sim = Simulator()
r = 10
sigma = 1
y = [0, 1]
n_reps = 3
output_dir = str(tmpdir)
sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)
shape_3d = (91, 109, 91)
shape_2d = (6, 238955)
y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None,index_col=None).T
flist = glob.glob(str(tmpdir.join('centered*.nii.gz')))
dat = Brain_Data(data=flist,Y=y)
# Test shape
assert dat.shape() == shape_2d
# Test Mean
assert dat.mean().shape()[0] == shape_2d[1]
# Test Std
assert dat.std().shape()[0] == shape_2d[1]
# Test to_nifti
d = dat.to_nifti
assert d().shape[0:3] == shape_3d
# # Test T-test
out = dat.ttest()
assert out['t'].shape()[0]==shape_2d[1]
# Test Regress
dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)),'X1':np.array(dat.Y).flatten()},index=None)
out = dat.regress()
assert out['beta'].shape() == (2,shape_2d[1])
# Test indexing
assert out['t'][1].shape()[0] == shape_2d[1]
# Test threshold
i=1
tt = threshold(out['t'][i], out['p'][i], threshold_dict={'fdr':.05})
assert tt.shape()[0] == shape_2d[1]
def simulate_data(n_observations, y, p, sigma, mask):
''' Simulate Brain Data
Args:
n_observations: (int) number of data points
y: (array) one dimensional array of signal
p: (float) probability of signal in voxels
sigma: (float) amount of gaussian noise to add
Returns:
data: (list) of Brain_Data objects
'''
dat = Brain_Data(mask).apply_mask(mask)
new_data = np.zeros((dat.shape()[0], n_observations))
for i in np.where(dat.data == 1)[0]:
if np.random.randint(0, high=10) < p:
new_data[i, :] = y
noise = np.random.randn(new_data.shape[0], n_observations) * sigma
dat.data = (new_data + noise).T
return dat
def test_data(tmpdir):
sim = Simulator()
r = 10
sigma = 1
y = [0, 1]
n_reps = 3
output_dir = str(tmpdir)
sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)
shape_3d = (91, 109, 91)
shape_2d = (6, 238955)
y = pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),
header=None,
index_col=None).T
flist = glob.glob(str(tmpdir.join('centered*.nii.gz')))
# Test load list
dat = Brain_Data(data=flist, Y=y)
# Test load file
assert Brain_Data(flist[0])
# Test to_nifti
d = dat.to_nifti()
assert d.shape[0:3] == shape_3d
# Test load nibabel
assert Brain_Data(d)
# Test shape
assert dat.shape() == shape_2d
# Test Mean
assert dat.mean().shape()[0] == shape_2d[1]
# Test Std
assert dat.std().shape()[0] == shape_2d[1]
# Test add
new = dat + dat
assert new.shape() == shape_2d
# Test subtract
new = dat - dat
assert new.shape() == shape_2d
# Test multiply
new = dat * dat
assert new.shape() == shape_2d
# Test Iterator
x = [x for x in dat]
assert len(x) == len(dat)
assert len(x[0].data.shape) == 1
# # Test T-test
out = dat.ttest()
assert out['t'].shape()[0] == shape_2d[1]
# # # Test T-test - permutation method
# out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1})
# assert out['t'].shape()[0]==shape_2d[1]
# Test Regress
dat.X = pd.DataFrame(
{
'Intercept': np.ones(len(dat.Y)),
'X1': np.array(dat.Y).flatten()
},
index=None)
out = dat.regress()
assert out['beta'].shape() == (2, shape_2d[1])
# Test indexing
assert out['t'][1].shape()[0] == shape_2d[1]
# Test threshold
i = 1
tt = threshold(out['t'][i], out['p'][i], .05)
assert isinstance(tt, Brain_Data)
# Test write
dat.write(os.path.join(str(tmpdir.join('test_write.nii'))))
assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii'))))
# Test append
assert dat.append(dat).shape()[0] == shape_2d[0] * 2
# Test distance
distance = dat.distance(method='euclidean')
assert distance.shape == (shape_2d[0], shape_2d[0])
# Test predict
stats = dat.predict(algorithm='svm',
cv_dict={
'type': 'kfolds',
'n_folds': 2,
'n': len(dat.Y)
},
plot=False,
**{'kernel': "linear"})
# Support Vector Regression, with 5 fold cross-validation with Platt Scaling
# This will output probabilities of each class
stats = dat.predict(algorithm='svm',
cv_dict=None,
plot=False,
**{
'kernel': 'linear',
'probability': True
})
assert isinstance(stats['weight_map'], Brain_Data)
# Logistic classificiation, with 5 fold stratified cross-validation.
stats = dat.predict(algorithm='logistic',
cv_dict={
'type': 'kfolds',
'n_folds': 5,
'n': len(dat.Y)
},
plot=False)
assert isinstance(stats['weight_map'], Brain_Data)
# Ridge classificiation, with 5 fold between-subject cross-validation, where data for each subject is held out together.
stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None, plot=False)
assert isinstance(stats['weight_map'], Brain_Data)
# Test Similarity
r = dat.similarity(stats['weight_map'])
assert len(r) == shape_2d[0]
r2 = dat.similarity(stats['weight_map'].to_nifti())
assert len(r2) == shape_2d[0]
# Test apply_mask - might move part of this to test mask suite
s1 = create_sphere([41, 64, 55], radius=10)
assert isinstance(s1, nb.Nifti1Image)
s2 = Brain_Data(s1)
masked_dat = dat.apply_mask(s1)
assert masked_dat.shape()[1] == np.sum(s2.data != 0)
# Test extract_roi
mask = create_sphere([41, 64, 55], radius=10)
assert len(dat.extract_roi(mask)) == shape_2d[0]
# Test r_to_z
z = dat.r_to_z()
assert z.shape() == dat.shape()
# Test copy
d_copy = dat.copy()
assert d_copy.shape() == dat.shape()
# Test detrend
detrend = dat.detrend()
assert detrend.shape() == dat.shape()
def get_trialtype_pain_regressors(self, nifti_data, onset_file):
print("importing nifti")
#import the nifti
#load the nltools prepped file if it's available.
if (os.path.isfile(nifti_data + "nltoolstandard.nii.gz")):
msmrl1 = Brain_Data(nifti_data + "nltoolstandard.nii.gz")
else: #but if it's not; no worries; just load the original one.
msmrl1 = Brain_Data(nifti_data + ".nii.gz")
msmrl1.write(nifti_data + "nltoolstandard.nii.gz")
#I want to standardize globally; this will preserve the relative strengths of each time point
#and preserve the relative activity at each voxel.
#and let's use the mean standard deviation across all the images.
#msmrl1.data = msmrl1.data - np.tile(msmrl1.mean().mean(),msmrl1.data.shape)
#msmrl1.data = msmrl1.data / np.tile(np.std(msmrl1.data,axis=1).mean(),msmrl1.data.shape)
# OR we could apply the standardization to the OUTPUT.
#grand_mean=msmrl1.mean().mean()
#grand_sd=np.std(msmrl1.data,axis=1).mean()
#preprocess the nifti?
print("importing onsets")
#import the onset
onsets = onsets_to_dm(onset_file, TR=2, runLength=msmrl1.shape()[0])
#process the onset files
#
onsets.sampling_rate = 2
onsets_convolved = onsets.convolve()
#delete columns with no information in them.
for c in onsets_convolved.columns:
if sum(onsets_convolved.ix[:, c]) <= 0:
print('deleting ' + str(c))
del onsets_convolved[c]
rowcount = onsets_convolved.__len__()
if rowcount != 360:
warnings.warn(
"Just a friendly FYI: expected number of rows is 360 but this subject had "
+ str(rowcount) +
". Probably this subject got cut off the task half-way through."
)
onsets_convolved['linearterm'] = range(1, rowcount + 1)
onsets_convolved['quadraticterm'] = [
pow(x, 2) for x in onsets_convolved['linearterm']
]
onsets_convolved['cubicterm'] = [
pow(x, 3) for x in onsets_convolved['linearterm']
]
onsets_convolved['ones'] = [1] * rowcount
msmrl1.X = onsets_convolved
print("convolved onsets; regressing...")
#regress the file on each of the onsets. So then, when we compare similarity to the regression, we'll be getting the
#regression to the each event, not to each TR.
regression = msmrl1.regress()
print("Regressing; calculating similarity to the pain map from " +
self.decoder_origin + "...")
msm_predicted_pain = regression['beta'].similarity(
self.decoder, 'dot_product')
msm_predicted_pain_scaled = msm_predicted_pain - msmrl1.data.std()
onset_colnames = onsets_convolved.columns.tolist()
msm_predicted_pain_dict = {}
for i, b in enumerate(msm_predicted_pain_scaled):
msm_predicted_pain_dict[onset_colnames[i]] = b
return msm_predicted_pain_dict
def test_brain_data(tmpdir):
# Add 3mm to list to test that resolution as well
for resolution in ['2mm']:
MNI_Template["resolution"] = resolution
sim = Simulator()
r = 10
sigma = 1
y = [0, 1]
n_reps = 3
output_dir = str(tmpdir)
dat = sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)
if MNI_Template["resolution"] == '2mm':
shape_3d = (91, 109, 91)
shape_2d = (6, 238955)
elif MNI_Template["resolution"] == '3mm':
shape_3d = (60, 72, 60)
shape_2d = (6, 71020)
y = pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None, index_col=None)
holdout = pd.read_csv(os.path.join(str(tmpdir.join('rep_id.csv'))),header=None,index_col=None)
# Test load list of 4D images
file_list = [str(tmpdir.join('data.nii.gz')), str(tmpdir.join('data.nii.gz'))]
dat = Brain_Data(file_list)
dat = Brain_Data([nb.load(x) for x in file_list])
# Test load list
dat = Brain_Data(data=str(tmpdir.join('data.nii.gz')), Y=y)
# Test concatenate
out = Brain_Data([x for x in dat])
assert isinstance(out, Brain_Data)
assert len(out)==len(dat)
# Test to_nifti
d = dat.to_nifti()
assert d.shape[0:3] == shape_3d
# Test load nibabel
assert Brain_Data(d)
# Test shape
assert dat.shape() == shape_2d
# Test Mean
assert dat.mean().shape()[0] == shape_2d[1]
# Test Std
assert dat.std().shape()[0] == shape_2d[1]
# Test add
new = dat + dat
assert new.shape() == shape_2d
# Test subtract
new = dat - dat
assert new.shape() == shape_2d
# Test multiply
new = dat * dat
assert new.shape() == shape_2d
# Test Indexing
index = [0, 3, 1]
assert len(dat[index]) == len(index)
index = range(4)
assert len(dat[index]) == len(index)
index = dat.Y == 1
assert len(dat[index.values.flatten()]) == index.values.sum()
assert len(dat[index]) == index.values.sum()
assert len(dat[:3]) == 3
# Test Iterator
x = [x for x in dat]
assert len(x) == len(dat)
assert len(x[0].data.shape) == 1
# # Test T-test
out = dat.ttest()
assert out['t'].shape()[0] == shape_2d[1]
# # # Test T-test - permutation method
# out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1})
# assert out['t'].shape()[0]==shape_2d[1]
# Test Regress
dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)),
'X1':np.array(dat.Y).flatten()}, index=None)
# Standard OLS
out = dat.regress()
assert type(out['beta'].data) == np.ndarray
assert type(out['t'].data) == np.ndarray
assert type(out['p'].data) == np.ndarray
assert type(out['residual'].data) == np.ndarray
assert type(out['df'].data) == np.ndarray
assert out['beta'].shape() == (2, shape_2d[1])
assert out['t'][1].shape()[0] == shape_2d[1]
# Robust OLS
out = dat.regress(mode='robust')
assert type(out['beta'].data) == np.ndarray
assert type(out['t'].data) == np.ndarray
assert type(out['p'].data) == np.ndarray
assert type(out['residual'].data) == np.ndarray
assert type(out['df'].data) == np.ndarray
assert out['beta'].shape() == (2, shape_2d[1])
assert out['t'][1].shape()[0] == shape_2d[1]
# Test threshold
i=1
tt = threshold(out['t'][i], out['p'][i], .05)
assert isinstance(tt, Brain_Data)
# Test write
dat.write(os.path.join(str(tmpdir.join('test_write.nii'))))
assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii'))))
# Test append
assert dat.append(dat).shape()[0] == shape_2d[0]*2
# Test distance
distance = dat.distance(method='euclidean')
assert isinstance(distance, Adjacency)
assert distance.square_shape()[0] == shape_2d[0]
# Test predict
stats = dat.predict(algorithm='svm',
cv_dict={'type': 'kfolds', 'n_folds': 2},
plot=False, **{'kernel':"linear"})
# Support Vector Regression, with 5 fold cross-validation with Platt Scaling
# This will output probabilities of each class
stats = dat.predict(algorithm='svm',
cv_dict=None, plot=False,
**{'kernel':'linear', 'probability':True})
assert isinstance(stats['weight_map'], Brain_Data)
# Logistic classificiation, with 2 fold cross-validation.
stats = dat.predict(algorithm='logistic',
cv_dict={'type': 'kfolds', 'n_folds': 2},
plot=False)
assert isinstance(stats['weight_map'], Brain_Data)
# Ridge classificiation,
stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None, plot=False)
assert isinstance(stats['weight_map'], Brain_Data)
# Ridge
stats = dat.predict(algorithm='ridge',
cv_dict={'type': 'kfolds', 'n_folds': 2,
'subject_id':holdout}, plot=False, **{'alpha':.1})
# Lasso
stats = dat.predict(algorithm='lasso',
cv_dict={'type': 'kfolds', 'n_folds': 2,
'stratified':dat.Y}, plot=False, **{'alpha':.1})
# PCR
stats = dat.predict(algorithm='pcr', cv_dict=None, plot=False)
# Test Similarity
r = dat.similarity(stats['weight_map'])
assert len(r) == shape_2d[0]
r2 = dat.similarity(stats['weight_map'].to_nifti())
assert len(r2) == shape_2d[0]
r = dat.similarity(stats['weight_map'], method='dot_product')
assert len(r) == shape_2d[0]
r = dat.similarity(stats['weight_map'], method='cosine')
assert len(r) == shape_2d[0]
r = dat.similarity(dat, method='correlation')
assert r.shape == (dat.shape()[0],dat.shape()[0])
r = dat.similarity(dat, method='dot_product')
assert r.shape == (dat.shape()[0],dat.shape()[0])
r = dat.similarity(dat, method='cosine')
assert r.shape == (dat.shape()[0],dat.shape()[0])
# Test apply_mask - might move part of this to test mask suite
s1 = create_sphere([12, 10, -8], radius=10)
assert isinstance(s1, nb.Nifti1Image)
masked_dat = dat.apply_mask(s1)
assert masked_dat.shape()[1] == np.sum(s1.get_data() != 0)
# Test extract_roi
mask = create_sphere([12, 10, -8], radius=10)
assert len(dat.extract_roi(mask)) == shape_2d[0]
# Test r_to_z
z = dat.r_to_z()
assert z.shape() == dat.shape()
# Test copy
d_copy = dat.copy()
assert d_copy.shape() == dat.shape()
# Test detrend
detrend = dat.detrend()
assert detrend.shape() == dat.shape()
# Test standardize
s = dat.standardize()
assert s.shape() == dat.shape()
assert np.isclose(np.sum(s.mean().data), 0, atol=.1)
s = dat.standardize(method='zscore')
assert s.shape() == dat.shape()
assert np.isclose(np.sum(s.mean().data), 0, atol=.1)
# Test Sum
s = dat.sum()
assert s.shape() == dat[1].shape()
# Test Groupby
s1 = create_sphere([12, 10, -8], radius=10)
s2 = create_sphere([22, -2, -22], radius=10)
mask = Brain_Data([s1, s2])
d = dat.groupby(mask)
assert isinstance(d, Groupby)
# Test Aggregate
mn = dat.aggregate(mask, 'mean')
assert isinstance(mn, Brain_Data)
assert len(mn.shape()) == 1
# Test Threshold
s1 = create_sphere([12, 10, -8], radius=10)
s2 = create_sphere([22, -2, -22], radius=10)
mask = Brain_Data(s1)*5
mask = mask + Brain_Data(s2)
m1 = mask.threshold(upper=.5)
m2 = mask.threshold(upper=3)
m3 = mask.threshold(upper='98%')
m4 = Brain_Data(s1)*5 + Brain_Data(s2)*-.5
m4 = mask.threshold(upper=.5,lower=-.3)
assert np.sum(m1.data > 0) > np.sum(m2.data > 0)
assert np.sum(m1.data > 0) == np.sum(m3.data > 0)
assert np.sum(m4.data[(m4.data > -.3) & (m4.data <.5)]) == 0
assert np.sum(m4.data[(m4.data < -.3) | (m4.data >.5)]) > 0
# Test Regions
r = mask.regions(min_region_size=10)
m1 = Brain_Data(s1)
m2 = r.threshold(1, binarize=True)
# assert len(r)==2
assert len(np.unique(r.to_nifti().get_data())) == 2
diff = m2-m1
assert np.sum(diff.data) == 0
# Test Bootstrap
masked = dat.apply_mask(create_sphere(radius=10, coordinates=[0, 0, 0]))
n_samples = 3
b = masked.bootstrap('mean', n_samples=n_samples)
assert isinstance(b['Z'], Brain_Data)
b = masked.bootstrap('std', n_samples=n_samples)
assert isinstance(b['Z'], Brain_Data)
b = masked.bootstrap('predict', n_samples=n_samples, plot=False)
assert isinstance(b['Z'], Brain_Data)
b = masked.bootstrap('predict', n_samples=n_samples,
plot=False, cv_dict={'type':'kfolds','n_folds':3})
assert isinstance(b['Z'], Brain_Data)
b = masked.bootstrap('predict', n_samples=n_samples,
save_weights=True, plot=False)
assert len(b['samples'])==n_samples
# Test decompose
n_components = 3
stats = dat.decompose(algorithm='pca', axis='voxels',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
stats = dat.decompose(algorithm='ica', axis='voxels',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
dat.data = dat.data + 2
dat.data[dat.data<0] = 0
stats = dat.decompose(algorithm='nnmf', axis='voxels',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
stats = dat.decompose(algorithm='fa', axis='voxels',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
stats = dat.decompose(algorithm='pca', axis='images',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
stats = dat.decompose(algorithm='ica', axis='images',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
dat.data = dat.data + 2
dat.data[dat.data<0] = 0
stats = dat.decompose(algorithm='nnmf', axis='images',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
stats = dat.decompose(algorithm='fa', axis='images',
n_components=n_components)
assert n_components == len(stats['components'])
assert stats['weights'].shape == (len(dat), n_components)
# Test Hyperalignment Method
sim = Simulator()
y = [0, 1]
n_reps = 10
s1 = create_sphere([0, 0, 0], radius=3)
d1 = sim.create_data(y, 1, reps=n_reps, output_dir=None).apply_mask(s1)
d2 = sim.create_data(y, 2, reps=n_reps, output_dir=None).apply_mask(s1)
d3 = sim.create_data(y, 3, reps=n_reps, output_dir=None).apply_mask(s1)
# Test procrustes using align
data = [d1, d2, d3]
out = align(data, method='procrustes')
assert len(data) == len(out['transformed'])
assert len(data) == len(out['transformation_matrix'])
assert data[0].shape() == out['common_model'].shape()
transformed = np.dot(d1.data, out['transformation_matrix'][0])
centered = d1.data - np.mean(d1.data, 0)
transformed = (np.dot(centered/np.linalg.norm(centered), out['transformation_matrix'][0])*out['scale'][0])
np.testing.assert_almost_equal(0, np.sum(out['transformed'][0].data - transformed), decimal=5)
# Test deterministic brain_data
bout = d1.align(out['common_model'], method='deterministic_srm')
assert d1.shape() == bout['transformed'].shape()
assert d1.shape() == bout['common_model'].shape()
assert d1.shape()[1] == bout['transformation_matrix'].shape[0]
btransformed = np.dot(d1.data, bout['transformation_matrix'])
np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data - btransformed))
# Test deterministic brain_data
bout = d1.align(out['common_model'], method='probabilistic_srm')
assert d1.shape() == bout['transformed'].shape()
assert d1.shape() == bout['common_model'].shape()
assert d1.shape()[1] == bout['transformation_matrix'].shape[0]
btransformed = np.dot(d1.data, bout['transformation_matrix'])
np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed))
# Test procrustes brain_data
bout = d1.align(out['common_model'], method='procrustes')
assert d1.shape() == bout['transformed'].shape()
assert d1.shape() == bout['common_model'].shape()
assert d1.shape()[1] == bout['transformation_matrix'].shape[0]
centered = d1.data - np.mean(d1.data, 0)
btransformed = (np.dot(centered/np.linalg.norm(centered), bout['transformation_matrix'])*bout['scale'])
np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed), decimal=5)
np.testing.assert_almost_equal(0, np.sum(out['transformed'][0].data - bout['transformed'].data))
# Test hyperalignment on Brain_Data over time (axis=1)
sim = Simulator()
y = [0, 1]
n_reps = 10
s1 = create_sphere([0, 0, 0], radius=5)
d1 = sim.create_data(y, 1, reps=n_reps, output_dir=None).apply_mask(s1)
d2 = sim.create_data(y, 2, reps=n_reps, output_dir=None).apply_mask(s1)
d3 = sim.create_data(y, 3, reps=n_reps, output_dir=None).apply_mask(s1)
data = [d1, d2, d3]
out = align(data, method='procrustes', axis=1)
assert len(data) == len(out['transformed'])
assert len(data) == len(out['transformation_matrix'])
assert data[0].shape() == out['common_model'].shape()
centered = data[0].data.T-np.mean(data[0].data.T, 0)
transformed = (np.dot(centered/np.linalg.norm(centered), out['transformation_matrix'][0])*out['scale'][0])
np.testing.assert_almost_equal(0,np.sum(out['transformed'][0].data-transformed.T), decimal=5)
bout = d1.align(out['common_model'], method='deterministic_srm', axis=1)
assert d1.shape() == bout['transformed'].shape()
assert d1.shape() == bout['common_model'].shape()
assert d1.shape()[0] == bout['transformation_matrix'].shape[0]
btransformed = np.dot(d1.data.T, bout['transformation_matrix'])
np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed.T))
bout = d1.align(out['common_model'], method='probabilistic_srm', axis=1)
assert d1.shape() == bout['transformed'].shape()
assert d1.shape() == bout['common_model'].shape()
assert d1.shape()[0] == bout['transformation_matrix'].shape[0]
btransformed = np.dot(d1.data.T, bout['transformation_matrix'])
np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed.T))
bout = d1.align(out['common_model'], method='procrustes', axis=1)
assert d1.shape() == bout['transformed'].shape()
assert d1.shape() == bout['common_model'].shape()
assert d1.shape()[0] == bout['transformation_matrix'].shape[0]
centered = d1.data.T-np.mean(d1.data.T, 0)
btransformed = (np.dot(centered/np.linalg.norm(centered), bout['transformation_matrix'])*bout['scale'])
np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed.T), decimal=5)
np.testing.assert_almost_equal(0, np.sum(out['transformed'][0].data-bout['transformed'].data))
f'{sub}_task-localizer_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz'
))
# Here are a few quick basic data operations.
#
# Find number of images in Brain_Data() instance
# In[49]:
print(len(data))
# Find the dimensions of the data (images x voxels)
# In[50]:
print(data.shape())
# We can use any type of indexing to slice the data such as integers, lists of integers, slices, or boolean vectors.
# In[51]:
import numpy as np
print(data[5].shape())
print(data[[1, 6, 2]].shape())
print(data[0:10].shape())
index = np.zeros(len(data), dtype=bool)
index[[1, 5, 9, 16, 20, 22]] = True
def import_and_convolve(self,
nifti_data,
onset_file,
data_mask=None,
motion_regressors=None):
print("importing nifti")
# import the nifti
# load the nltools prepped file if it's available.
if (os.path.isfile(nifti_data + self.data_fmri_space + ".nii.gz")):
msmrl1 = Brain_Data(nifti_data + self.data_fmri_space + ".nii.gz",
mask=data_mask)
else: # but if it's not; no worries; just load the original one.
msmrl1 = Brain_Data(nifti_data + ".nii.gz", mask=data_mask)
msmrl1.write(nifti_data + self.data_fmri_space + ".nii.gz")
# I want to standardize globally; this will preserve the relative strengths of each time point
# and preserve the relative activity at each voxel.
# and let's use the mean standard deviation across all the images.
# msmrl1.data = msmrl1.data - np.tile(msmrl1.mean().mean(),msmrl1.data.shape)
# msmrl1.data = msmrl1.data / np.tile(np.std(msmrl1.data,axis=1).mean(),msmrl1.data.shape)
# OR we could apply the standardization to the OUTPUT.
# grand_mean=msmrl1.mean().mean()
# grand_sd=np.std(msmrl1.data,axis=1).mean()
# preprocess the nifti?
print("importing onsets")
# import the onset
onsets = onsets_to_dm(onset_file, TR=2, runLength=msmrl1.shape()[0])
# process the onset files
#
onsets.sampling_rate = 2
onsets_convolved = onsets.convolve()
# delete columns with no information in them.
for c in onsets_convolved.columns:
if sum(onsets_convolved.ix[:, c]) <= 0:
print('deleting ' + str(c))
del onsets_convolved[c]
rowcount = onsets_convolved.__len__()
if rowcount != 360:
warnings.warn(
"Just a friendly FYI: expected number of rows is 360 but this subject had "
+ str(rowcount) +
". Probably this subject got cut off the task half-way through."
)
onsets_convolved['linearterm'] = range(1, rowcount + 1)
onsets_convolved['quadraticterm'] = [
pow(x, 2) for x in onsets_convolved['linearterm']
]
onsets_convolved['cubicterm'] = [
pow(x, 3) for x in onsets_convolved['linearterm']
]
onsets_convolved['ones'] = [1] * rowcount
if (motion_regressors is not None):
onsets_convolved = pandas.concat(
[onsets_convolved, motion_regressors], axis=1)
msmrl1.X = onsets_convolved
return msmrl1
def test_brain_data(tmpdir):
sim = Simulator()
r = 10
sigma = 1
y = [0, 1]
n_reps = 3
output_dir = str(tmpdir)
sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir)
shape_3d = (91, 109, 91)
shape_2d = (6, 238955)
y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))), header=None,index_col=None).T
holdout=pd.read_csv(os.path.join(str(tmpdir.join('rep_id.csv'))),header=None,index_col=None).T
flist = glob.glob(str(tmpdir.join('centered*.nii.gz')))
# Test load list
dat = Brain_Data(data=flist,Y=y)
# Test load file
assert Brain_Data(flist[0])
# Test to_nifti
d = dat.to_nifti()
assert d.shape[0:3] == shape_3d
# Test load nibabel
assert Brain_Data(d)
# Test shape
assert dat.shape() == shape_2d
# Test Mean
assert dat.mean().shape()[0] == shape_2d[1]
# Test Std
assert dat.std().shape()[0] == shape_2d[1]
# Test add
new = dat + dat
assert new.shape() == shape_2d
# Test subtract
new = dat - dat
assert new.shape() == shape_2d
# Test multiply
new = dat * dat
assert new.shape() == shape_2d
# Test Iterator
x = [x for x in dat]
assert len(x) == len(dat)
assert len(x[0].data.shape) == 1
# # Test T-test
out = dat.ttest()
assert out['t'].shape()[0] == shape_2d[1]
# # # Test T-test - permutation method
# out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1})
# assert out['t'].shape()[0]==shape_2d[1]
# Test Regress
dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)), 'X1':np.array(dat.Y).flatten()},index=None)
out = dat.regress()
assert out['beta'].shape() == (2,shape_2d[1])
# Test indexing
assert out['t'][1].shape()[0] == shape_2d[1]
# Test threshold
i=1
tt = threshold(out['t'][i], out['p'][i], .05)
assert isinstance(tt,Brain_Data)
# Test write
dat.write(os.path.join(str(tmpdir.join('test_write.nii'))))
assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii'))))
# Test append
assert dat.append(dat).shape()[0]==shape_2d[0]*2
# Test distance
distance = dat.distance(method='euclidean')
assert isinstance(distance,Adjacency)
assert distance.square_shape()[0]==shape_2d[0]
# Test predict
stats = dat.predict(algorithm='svm', cv_dict={'type': 'kfolds','n_folds': 2}, plot=False,**{'kernel':"linear"})
# Support Vector Regression, with 5 fold cross-validation with Platt Scaling
# This will output probabilities of each class
stats = dat.predict(algorithm='svm', cv_dict=None, plot=False,**{'kernel':'linear', 'probability':True})
assert isinstance(stats['weight_map'],Brain_Data)
# Logistic classificiation, with 2 fold cross-validation.
stats = dat.predict(algorithm='logistic', cv_dict={'type': 'kfolds', 'n_folds': 2}, plot=False)
assert isinstance(stats['weight_map'],Brain_Data)
# Ridge classificiation,
stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None,plot=False)
assert isinstance(stats['weight_map'],Brain_Data)
# Ridge
stats = dat.predict(algorithm='ridge', cv_dict={'type': 'kfolds', 'n_folds': 2,'subject_id':holdout}, plot=False,**{'alpha':.1})
# Lasso
stats = dat.predict(algorithm='lasso', cv_dict={'type': 'kfolds', 'n_folds': 2,'stratified':dat.Y}, plot=False,**{'alpha':.1})
# PCR
stats = dat.predict(algorithm='pcr', cv_dict=None, plot=False)
# Test Similarity
r = dat.similarity(stats['weight_map'])
assert len(r) == shape_2d[0]
r2 = dat.similarity(stats['weight_map'].to_nifti())
assert len(r2) == shape_2d[0]
# Test apply_mask - might move part of this to test mask suite
s1 = create_sphere([12, 10, -8], radius=10)
assert isinstance(s1, nb.Nifti1Image)
s2 = Brain_Data(s1)
masked_dat = dat.apply_mask(s1)
assert masked_dat.shape()[1] == np.sum(s2.data != 0)
# Test extract_roi
mask = create_sphere([12, 10, -8], radius=10)
assert len(dat.extract_roi(mask)) == shape_2d[0]
# Test r_to_z
z = dat.r_to_z()
assert z.shape() == dat.shape()
# Test copy
d_copy = dat.copy()
assert d_copy.shape() == dat.shape()
# Test detrend
detrend = dat.detrend()
assert detrend.shape() == dat.shape()
# Test standardize
s = dat.standardize()
assert s.shape() == dat.shape()
assert np.isclose(np.sum(s.mean().data), 0, atol=.1)
s = dat.standardize(method='zscore')
assert s.shape() == dat.shape()
assert np.isclose(np.sum(s.mean().data), 0, atol=.1)
# Test Sum
s = dat.sum()
assert s.shape() == dat[1].shape()
# Test Groupby
s1 = create_sphere([12, 10, -8], radius=10)
s2 = create_sphere([22, -2, -22], radius=10)
mask = Brain_Data([s1, s2])
d = dat.groupby(mask)
assert isinstance(d, Groupby)
# Test Aggregate
mn = dat.aggregate(mask, 'mean')
assert isinstance(mn, Brain_Data)
assert len(mn.shape()) == 1
# Test Threshold
s1 = create_sphere([12, 10, -8], radius=10)
s2 = create_sphere([22, -2, -22], radius=10)
mask = Brain_Data(s1)*5
mask = mask + Brain_Data(s2)
m1 = mask.threshold(thresh=.5)
m2 = mask.threshold(thresh=3)
m3 = mask.threshold(thresh='98%')
assert np.sum(m1.data > 0) > np.sum(m2.data > 0)
assert np.sum(m1.data > 0) == np.sum(m3.data > 0)
# Test Regions
r = mask.regions(min_region_size=10)
m1 = Brain_Data(s1)
m2 = r.threshold(1, binarize=True)
# assert len(r)==2
assert len(np.unique(r.to_nifti().get_data())) == 2 # JC edit: I think this is what you were trying to do
diff = m2-m1
assert np.sum(diff.data) == 0
#need to regress on this value PainLevel to build our regression of Pain
#we have number of subjects and we ought to build a model where
#we correlate pain within-subject, then average across subjects
#or possibly a linear model in which we regress out each subject's mean then calculate across subjects
#or we could run a bunch of within-subject correlations of pain-level with voxel activity
#then train on the 84/3=18 images.
#ideally a linear repeated-measures model or something
#but Luke
#sub113
sub113_onset_file='/Users/benjaminsmith/GDrive/joint-modeling/reversal-learning/behavioral-analysis/data/runfiles/runfilepunishmentcompare20170819T170218_s113_punishment_r1.txt'
msmrl1=Brain_Data('/Users/benjaminsmith/GDrive/joint-modeling/reversal-learning/behavioral-analysis/data/preprocessed/sub113/ReversalLearningPunishrun1.nii.gz')
type(msmrl1)
msmrl1.shape()
datamean=np.mean(msmrl1.data,axis=1)
with sns.plotting_context(context='paper',font_scale=2):
sns.factorplot(data=pd.DataFrame(data=
{'ImageMean':datamean,
'Timepoint':range(1,361)})
,x='Timepoint',y='ImageMean')
msmrl1_demeaned=msmrl1-datamean
datamean.shape()
msmrl1.shape()
data_demeaned_mean=np.mean(msmrl1.data,axis=1)
print(preproc_dir, subject_id, episode)
sub_cov, sub_epi = fileGetter(preproc_dir,subject_id,episode)
#Load run data
print("Loading brain data: {}".format(smooth))
dat = Brain_Data(sub_epi)
cov_mat = Design_Matrix(pd.read_csv(sub_cov[0]).fillna(0), sampling_rate=TR)
# Add Intercept
cov_mat['Intercept'] = 1
# Add Linear Trend
cov_mat['LinearTrend'] = range(cov_mat.shape[0])-np.mean(range(cov_mat.shape[0]))
cov_mat['QuadraticTrend'] = cov_mat['LinearTrend']**2
cov_mat['CSF'] = dat.extract_roi(csf.threshold(.85,binarize=True))
assert cov_mat.shape[0] == dat.shape()[0]
spikeless_idx = np.logical_not( startswith(cov_mat.columns.values.astype(str), "spike") | startswith(cov_mat.columns.values.astype(str), "FD") )
#dat.X = cov_mat
dat.X = cov_mat.loc[:,spikeless_idx]
datcln = dat.regress()['residual']
# ## Loop through voxels to produce STPs
# In[14]:
zero_voxels = np.apply_along_axis(lambda x: np.count_nonzero(x) == 0, axis=0, arr=datcln.data)
zero_voxels = np.arange(datcln.data.shape[1])[zero_voxels]
