def check_brain_data(data, mask=None): """Check if data is a Brain_Data Instance.""" from nltools.data import Brain_Data if not isinstance(data, Brain_Data): if isinstance(data, nib.Nifti1Image): data = Brain_Data(data, mask=mask) else: raise ValueError("Make sure data is a Brain_Data instance.") else: if mask is not None: data = data.apply_mask(mask) return data
def test_data(tmpdir): sim = Simulator() r = 10 sigma = 1 y = [0, 1] n_reps = 3 output_dir = str(tmpdir) sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir) shape_3d = (91, 109, 91) shape_2d = (6, 238955) y = pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))), header=None, index_col=None).T flist = glob.glob(str(tmpdir.join('centered*.nii.gz'))) # Test load list dat = Brain_Data(data=flist, Y=y) # Test load file assert Brain_Data(flist[0]) # Test to_nifti d = dat.to_nifti() assert d.shape[0:3] == shape_3d # Test load nibabel assert Brain_Data(d) # Test shape assert dat.shape() == shape_2d # Test Mean assert dat.mean().shape()[0] == shape_2d[1] # Test Std assert dat.std().shape()[0] == shape_2d[1] # Test add new = dat + dat assert new.shape() == shape_2d # Test subtract new = dat - dat assert new.shape() == shape_2d # Test multiply new = dat * dat assert new.shape() == shape_2d # Test Iterator x = [x for x in dat] assert len(x) == len(dat) assert len(x[0].data.shape) == 1 # # Test T-test out = dat.ttest() assert out['t'].shape()[0] == shape_2d[1] # # # Test T-test - permutation method # out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1}) # assert out['t'].shape()[0]==shape_2d[1] # Test Regress dat.X = pd.DataFrame( { 'Intercept': np.ones(len(dat.Y)), 'X1': np.array(dat.Y).flatten() }, index=None) out = dat.regress() assert out['beta'].shape() == (2, shape_2d[1]) # Test indexing assert out['t'][1].shape()[0] == shape_2d[1] # Test threshold i = 1 tt = threshold(out['t'][i], out['p'][i], .05) assert isinstance(tt, Brain_Data) # Test write dat.write(os.path.join(str(tmpdir.join('test_write.nii')))) assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii')))) # Test append assert dat.append(dat).shape()[0] == shape_2d[0] * 2 # Test distance distance = dat.distance(method='euclidean') assert distance.shape == (shape_2d[0], shape_2d[0]) # Test predict stats = dat.predict(algorithm='svm', cv_dict={ 'type': 'kfolds', 'n_folds': 2, 'n': len(dat.Y) }, plot=False, **{'kernel': "linear"}) # Support Vector Regression, with 5 fold cross-validation with Platt Scaling # This will output probabilities of each class stats = dat.predict(algorithm='svm', cv_dict=None, plot=False, **{ 'kernel': 'linear', 'probability': True }) assert isinstance(stats['weight_map'], Brain_Data) # Logistic classificiation, with 5 fold stratified cross-validation. stats = dat.predict(algorithm='logistic', cv_dict={ 'type': 'kfolds', 'n_folds': 5, 'n': len(dat.Y) }, plot=False) assert isinstance(stats['weight_map'], Brain_Data) # Ridge classificiation, with 5 fold between-subject cross-validation, where data for each subject is held out together. stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None, plot=False) assert isinstance(stats['weight_map'], Brain_Data) # Test Similarity r = dat.similarity(stats['weight_map']) assert len(r) == shape_2d[0] r2 = dat.similarity(stats['weight_map'].to_nifti()) assert len(r2) == shape_2d[0] # Test apply_mask - might move part of this to test mask suite s1 = create_sphere([41, 64, 55], radius=10) assert isinstance(s1, nb.Nifti1Image) s2 = Brain_Data(s1) masked_dat = dat.apply_mask(s1) assert masked_dat.shape()[1] == np.sum(s2.data != 0) # Test extract_roi mask = create_sphere([41, 64, 55], radius=10) assert len(dat.extract_roi(mask)) == shape_2d[0] # Test r_to_z z = dat.r_to_z() assert z.shape() == dat.shape() # Test copy d_copy = dat.copy() assert d_copy.shape() == dat.shape() # Test detrend detrend = dat.detrend() assert detrend.shape() == dat.shape()
def train_model(image_list, algorithm, cross_validation, output_dir, file_path_key='resampled_file', mask=None): """ :param image_list: A list of dictionaries of the form { 'collection_id': '504', 'filename': 'Pain_Subject_1_Low.nii.gz', 'target': '1', 'resampled_file': 'path/to/the/resampled/file.nii.gz', 'original_file': 'path/to/the/original/file.nii.gz' } """ tic = time.time() # Start Timer try: holdout = [int(item['subject_id']) for item in image_list] except KeyError: holdout = None if cross_validation: if holdout: cross_validation['subject_id'] = holdout elif cross_validation['type'] == 'loso': raise ValueError( "subject_id is required for a LOSO cross validation.") extra = {} if algorithm in ('svr', 'svm'): extra = {'kernel': 'linear'} categorical_mapping = None if algorithm in CLASSIFICATION_ALGORITHMS: classes = {item['target'] for item in image_list} assert len(classes) == 2, ('More than two classes. ' 'Classification requires binary data.') categorical_mapping = {cls: index for index, cls in enumerate(classes)} for image in image_list: image['target'] = categorical_mapping[image['target']] Y = pd.DataFrame([float(item['target']) for item in image_list]) file_path_list = [item[file_path_key] for item in image_list] dat = Brain_Data(data=file_path_list, Y=Y) if mask: log.info('Applying a mask') nifti_mask = nb.load(mask) dat = dat.apply_mask(nifti_mask) output = dat.predict(algorithm=algorithm, cv_dict=cross_validation, plot=True, **extra) weightmap_filename = '%s_weightmap.nii.gz' % algorithm output['weight_map'].write(os.path.join(output_dir, weightmap_filename)) log.info("Elapsed: %.2f seconds", (time.time() - tic)) # Stop timer result = { 'weightmap': weightmap_filename, 'intercept': float(output['intercept']), 'scatterplot': '%s_scatterplot.png ' % algorithm, 'stats': { key: output[key].tolist() for key in ('Y', 'yfit_xval', 'yfit_all') if key in output }, 'categorical_mapping': categorical_mapping, 'summary': get_summary(output) } if 'roc' in output: result['roc'] = output['roc'] return result
mask = Brain_Data(os.path.join('..', 'masks', 'k50_2mm.nii.gz')) mask_x = expand_mask(mask) mask.plot() # Ok, now we will want to calculate the pattern similar within each ROI across the 10 conditions. # # We will loop over each ROI and extract the pattern data across all conditions and then compute the correlation distance between each condition. This data will now be an `Adjacency` object that we discussed in the Lab 13: Connectivity. We will temporarily store this in a list. # # Notice that for each iteration of the loop we apply the ROI mask to our beta images and then calculate the correlation distance. # In[79]: out = [] for m in mask_x: out.append(beta.apply_mask(m).distance(metric='correlation')) # Let's plot an example ROI and it's associated distance matrix. # # Here is a left motor parcel. Notice how the distance is small between the motor left auditory and visual and motor right auditory and visual beta images? # In[83]: roi = 26 plot_glass_brain(mask_x[roi].to_nifti()) out[roi].labels = conditions f2 = out[roi].plot(vmin=0, vmax=2, cmap='RdBu_r') # We can also visualize this distance matrix using multidimensional scaling with the `plot_mds()` method. This method projects the images into either a 2D or 3D plane for ease of visualization. There are many other distance based projection methods such as T-SNE or UMAP, we encourage readers to check out the excellent [hypertools](https://hypertools.readthedocs.io/en/latest/) package that has a great implementation of all of these methods. # # Notice how the motor right visual and auditory are near each other, while the motor left visual and auditory are grouped together further away?
def test_brain_data(tmpdir): # Add 3mm to list to test that resolution as well for resolution in ['2mm']: MNI_Template["resolution"] = resolution sim = Simulator() r = 10 sigma = 1 y = [0, 1] n_reps = 3 output_dir = str(tmpdir) dat = sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir) if MNI_Template["resolution"] == '2mm': shape_3d = (91, 109, 91) shape_2d = (6, 238955) elif MNI_Template["resolution"] == '3mm': shape_3d = (60, 72, 60) shape_2d = (6, 71020) y = pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))),header=None, index_col=None) holdout = pd.read_csv(os.path.join(str(tmpdir.join('rep_id.csv'))),header=None,index_col=None) # Test load list of 4D images file_list = [str(tmpdir.join('data.nii.gz')), str(tmpdir.join('data.nii.gz'))] dat = Brain_Data(file_list) dat = Brain_Data([nb.load(x) for x in file_list]) # Test load list dat = Brain_Data(data=str(tmpdir.join('data.nii.gz')), Y=y) # Test concatenate out = Brain_Data([x for x in dat]) assert isinstance(out, Brain_Data) assert len(out)==len(dat) # Test to_nifti d = dat.to_nifti() assert d.shape[0:3] == shape_3d # Test load nibabel assert Brain_Data(d) # Test shape assert dat.shape() == shape_2d # Test Mean assert dat.mean().shape()[0] == shape_2d[1] # Test Std assert dat.std().shape()[0] == shape_2d[1] # Test add new = dat + dat assert new.shape() == shape_2d # Test subtract new = dat - dat assert new.shape() == shape_2d # Test multiply new = dat * dat assert new.shape() == shape_2d # Test Indexing index = [0, 3, 1] assert len(dat[index]) == len(index) index = range(4) assert len(dat[index]) == len(index) index = dat.Y == 1 assert len(dat[index.values.flatten()]) == index.values.sum() assert len(dat[index]) == index.values.sum() assert len(dat[:3]) == 3 # Test Iterator x = [x for x in dat] assert len(x) == len(dat) assert len(x[0].data.shape) == 1 # # Test T-test out = dat.ttest() assert out['t'].shape()[0] == shape_2d[1] # # # Test T-test - permutation method # out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1}) # assert out['t'].shape()[0]==shape_2d[1] # Test Regress dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)), 'X1':np.array(dat.Y).flatten()}, index=None) # Standard OLS out = dat.regress() assert type(out['beta'].data) == np.ndarray assert type(out['t'].data) == np.ndarray assert type(out['p'].data) == np.ndarray assert type(out['residual'].data) == np.ndarray assert type(out['df'].data) == np.ndarray assert out['beta'].shape() == (2, shape_2d[1]) assert out['t'][1].shape()[0] == shape_2d[1] # Robust OLS out = dat.regress(mode='robust') assert type(out['beta'].data) == np.ndarray assert type(out['t'].data) == np.ndarray assert type(out['p'].data) == np.ndarray assert type(out['residual'].data) == np.ndarray assert type(out['df'].data) == np.ndarray assert out['beta'].shape() == (2, shape_2d[1]) assert out['t'][1].shape()[0] == shape_2d[1] # Test threshold i=1 tt = threshold(out['t'][i], out['p'][i], .05) assert isinstance(tt, Brain_Data) # Test write dat.write(os.path.join(str(tmpdir.join('test_write.nii')))) assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii')))) # Test append assert dat.append(dat).shape()[0] == shape_2d[0]*2 # Test distance distance = dat.distance(method='euclidean') assert isinstance(distance, Adjacency) assert distance.square_shape()[0] == shape_2d[0] # Test predict stats = dat.predict(algorithm='svm', cv_dict={'type': 'kfolds', 'n_folds': 2}, plot=False, **{'kernel':"linear"}) # Support Vector Regression, with 5 fold cross-validation with Platt Scaling # This will output probabilities of each class stats = dat.predict(algorithm='svm', cv_dict=None, plot=False, **{'kernel':'linear', 'probability':True}) assert isinstance(stats['weight_map'], Brain_Data) # Logistic classificiation, with 2 fold cross-validation. stats = dat.predict(algorithm='logistic', cv_dict={'type': 'kfolds', 'n_folds': 2}, plot=False) assert isinstance(stats['weight_map'], Brain_Data) # Ridge classificiation, stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None, plot=False) assert isinstance(stats['weight_map'], Brain_Data) # Ridge stats = dat.predict(algorithm='ridge', cv_dict={'type': 'kfolds', 'n_folds': 2, 'subject_id':holdout}, plot=False, **{'alpha':.1}) # Lasso stats = dat.predict(algorithm='lasso', cv_dict={'type': 'kfolds', 'n_folds': 2, 'stratified':dat.Y}, plot=False, **{'alpha':.1}) # PCR stats = dat.predict(algorithm='pcr', cv_dict=None, plot=False) # Test Similarity r = dat.similarity(stats['weight_map']) assert len(r) == shape_2d[0] r2 = dat.similarity(stats['weight_map'].to_nifti()) assert len(r2) == shape_2d[0] r = dat.similarity(stats['weight_map'], method='dot_product') assert len(r) == shape_2d[0] r = dat.similarity(stats['weight_map'], method='cosine') assert len(r) == shape_2d[0] r = dat.similarity(dat, method='correlation') assert r.shape == (dat.shape()[0],dat.shape()[0]) r = dat.similarity(dat, method='dot_product') assert r.shape == (dat.shape()[0],dat.shape()[0]) r = dat.similarity(dat, method='cosine') assert r.shape == (dat.shape()[0],dat.shape()[0]) # Test apply_mask - might move part of this to test mask suite s1 = create_sphere([12, 10, -8], radius=10) assert isinstance(s1, nb.Nifti1Image) masked_dat = dat.apply_mask(s1) assert masked_dat.shape()[1] == np.sum(s1.get_data() != 0) # Test extract_roi mask = create_sphere([12, 10, -8], radius=10) assert len(dat.extract_roi(mask)) == shape_2d[0] # Test r_to_z z = dat.r_to_z() assert z.shape() == dat.shape() # Test copy d_copy = dat.copy() assert d_copy.shape() == dat.shape() # Test detrend detrend = dat.detrend() assert detrend.shape() == dat.shape() # Test standardize s = dat.standardize() assert s.shape() == dat.shape() assert np.isclose(np.sum(s.mean().data), 0, atol=.1) s = dat.standardize(method='zscore') assert s.shape() == dat.shape() assert np.isclose(np.sum(s.mean().data), 0, atol=.1) # Test Sum s = dat.sum() assert s.shape() == dat[1].shape() # Test Groupby s1 = create_sphere([12, 10, -8], radius=10) s2 = create_sphere([22, -2, -22], radius=10) mask = Brain_Data([s1, s2]) d = dat.groupby(mask) assert isinstance(d, Groupby) # Test Aggregate mn = dat.aggregate(mask, 'mean') assert isinstance(mn, Brain_Data) assert len(mn.shape()) == 1 # Test Threshold s1 = create_sphere([12, 10, -8], radius=10) s2 = create_sphere([22, -2, -22], radius=10) mask = Brain_Data(s1)*5 mask = mask + Brain_Data(s2) m1 = mask.threshold(upper=.5) m2 = mask.threshold(upper=3) m3 = mask.threshold(upper='98%') m4 = Brain_Data(s1)*5 + Brain_Data(s2)*-.5 m4 = mask.threshold(upper=.5,lower=-.3) assert np.sum(m1.data > 0) > np.sum(m2.data > 0) assert np.sum(m1.data > 0) == np.sum(m3.data > 0) assert np.sum(m4.data[(m4.data > -.3) & (m4.data <.5)]) == 0 assert np.sum(m4.data[(m4.data < -.3) | (m4.data >.5)]) > 0 # Test Regions r = mask.regions(min_region_size=10) m1 = Brain_Data(s1) m2 = r.threshold(1, binarize=True) # assert len(r)==2 assert len(np.unique(r.to_nifti().get_data())) == 2 diff = m2-m1 assert np.sum(diff.data) == 0 # Test Bootstrap masked = dat.apply_mask(create_sphere(radius=10, coordinates=[0, 0, 0])) n_samples = 3 b = masked.bootstrap('mean', n_samples=n_samples) assert isinstance(b['Z'], Brain_Data) b = masked.bootstrap('std', n_samples=n_samples) assert isinstance(b['Z'], Brain_Data) b = masked.bootstrap('predict', n_samples=n_samples, plot=False) assert isinstance(b['Z'], Brain_Data) b = masked.bootstrap('predict', n_samples=n_samples, plot=False, cv_dict={'type':'kfolds','n_folds':3}) assert isinstance(b['Z'], Brain_Data) b = masked.bootstrap('predict', n_samples=n_samples, save_weights=True, plot=False) assert len(b['samples'])==n_samples # Test decompose n_components = 3 stats = dat.decompose(algorithm='pca', axis='voxels', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) stats = dat.decompose(algorithm='ica', axis='voxels', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) dat.data = dat.data + 2 dat.data[dat.data<0] = 0 stats = dat.decompose(algorithm='nnmf', axis='voxels', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) stats = dat.decompose(algorithm='fa', axis='voxels', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) stats = dat.decompose(algorithm='pca', axis='images', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) stats = dat.decompose(algorithm='ica', axis='images', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) dat.data = dat.data + 2 dat.data[dat.data<0] = 0 stats = dat.decompose(algorithm='nnmf', axis='images', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) stats = dat.decompose(algorithm='fa', axis='images', n_components=n_components) assert n_components == len(stats['components']) assert stats['weights'].shape == (len(dat), n_components) # Test Hyperalignment Method sim = Simulator() y = [0, 1] n_reps = 10 s1 = create_sphere([0, 0, 0], radius=3) d1 = sim.create_data(y, 1, reps=n_reps, output_dir=None).apply_mask(s1) d2 = sim.create_data(y, 2, reps=n_reps, output_dir=None).apply_mask(s1) d3 = sim.create_data(y, 3, reps=n_reps, output_dir=None).apply_mask(s1) # Test procrustes using align data = [d1, d2, d3] out = align(data, method='procrustes') assert len(data) == len(out['transformed']) assert len(data) == len(out['transformation_matrix']) assert data[0].shape() == out['common_model'].shape() transformed = np.dot(d1.data, out['transformation_matrix'][0]) centered = d1.data - np.mean(d1.data, 0) transformed = (np.dot(centered/np.linalg.norm(centered), out['transformation_matrix'][0])*out['scale'][0]) np.testing.assert_almost_equal(0, np.sum(out['transformed'][0].data - transformed), decimal=5) # Test deterministic brain_data bout = d1.align(out['common_model'], method='deterministic_srm') assert d1.shape() == bout['transformed'].shape() assert d1.shape() == bout['common_model'].shape() assert d1.shape()[1] == bout['transformation_matrix'].shape[0] btransformed = np.dot(d1.data, bout['transformation_matrix']) np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data - btransformed)) # Test deterministic brain_data bout = d1.align(out['common_model'], method='probabilistic_srm') assert d1.shape() == bout['transformed'].shape() assert d1.shape() == bout['common_model'].shape() assert d1.shape()[1] == bout['transformation_matrix'].shape[0] btransformed = np.dot(d1.data, bout['transformation_matrix']) np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed)) # Test procrustes brain_data bout = d1.align(out['common_model'], method='procrustes') assert d1.shape() == bout['transformed'].shape() assert d1.shape() == bout['common_model'].shape() assert d1.shape()[1] == bout['transformation_matrix'].shape[0] centered = d1.data - np.mean(d1.data, 0) btransformed = (np.dot(centered/np.linalg.norm(centered), bout['transformation_matrix'])*bout['scale']) np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed), decimal=5) np.testing.assert_almost_equal(0, np.sum(out['transformed'][0].data - bout['transformed'].data)) # Test hyperalignment on Brain_Data over time (axis=1) sim = Simulator() y = [0, 1] n_reps = 10 s1 = create_sphere([0, 0, 0], radius=5) d1 = sim.create_data(y, 1, reps=n_reps, output_dir=None).apply_mask(s1) d2 = sim.create_data(y, 2, reps=n_reps, output_dir=None).apply_mask(s1) d3 = sim.create_data(y, 3, reps=n_reps, output_dir=None).apply_mask(s1) data = [d1, d2, d3] out = align(data, method='procrustes', axis=1) assert len(data) == len(out['transformed']) assert len(data) == len(out['transformation_matrix']) assert data[0].shape() == out['common_model'].shape() centered = data[0].data.T-np.mean(data[0].data.T, 0) transformed = (np.dot(centered/np.linalg.norm(centered), out['transformation_matrix'][0])*out['scale'][0]) np.testing.assert_almost_equal(0,np.sum(out['transformed'][0].data-transformed.T), decimal=5) bout = d1.align(out['common_model'], method='deterministic_srm', axis=1) assert d1.shape() == bout['transformed'].shape() assert d1.shape() == bout['common_model'].shape() assert d1.shape()[0] == bout['transformation_matrix'].shape[0] btransformed = np.dot(d1.data.T, bout['transformation_matrix']) np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed.T)) bout = d1.align(out['common_model'], method='probabilistic_srm', axis=1) assert d1.shape() == bout['transformed'].shape() assert d1.shape() == bout['common_model'].shape() assert d1.shape()[0] == bout['transformation_matrix'].shape[0] btransformed = np.dot(d1.data.T, bout['transformation_matrix']) np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed.T)) bout = d1.align(out['common_model'], method='procrustes', axis=1) assert d1.shape() == bout['transformed'].shape() assert d1.shape() == bout['common_model'].shape() assert d1.shape()[0] == bout['transformation_matrix'].shape[0] centered = d1.data.T-np.mean(d1.data.T, 0) btransformed = (np.dot(centered/np.linalg.norm(centered), bout['transformation_matrix'])*bout['scale']) np.testing.assert_almost_equal(0, np.sum(bout['transformed'].data-btransformed.T), decimal=5) np.testing.assert_almost_equal(0, np.sum(out['transformed'][0].data-bout['transformed'].data))
img_BD.plot(anatomical=fp_anat, title=" - ".join(img_title + ['BDwithMask']), output_file="_".join(img_title + ['BDwithMask'])) # read in subject-specific roi.csv file subj_roi_df = pd.read_csv( str(pathlib.PurePath(workdir, dirname, 'LRTC_roi_10mm.csv'))) # iterating roi.csv file within each subject's folder for row in subj_roi_df.itertuples(): # saving and returning the roi mask created mask = roi_mask(row.x, row.y, row.z, row.label, row.size) # applying the roi mask on image of interest roi_img = img_BD.apply_mask(mask) # concatenate extracted neurite array into dataframe extract_df = pd.concat([ extract_df, pd.DataFrame(roi_img.data, columns=[dirname + '_' + row.label]) ], axis=1) # ortho views at roi coordinates on subject's brain instead of default axial plots # masked_img.plot() fig_label = "{0}, {1} mm sphere at [{2},{3},{4}]".format( row.label, str(row.size), row.x, row.y, row.z) file_label = "{0}_{1}_{2}mm_sphere.png".format(dirname, row.label, str(row.size))
plotting.plot_roi(rois, bg_img=anat, display_mode='z', dim=-1) # %% visualizing Scan2-Scan1 neurite density difference map sub = image.load_img(sub_img) plotting.plot_roi(sub, bg_img=anat, display_mode='z', dim=-1, threshold=0.5) # %% plot the masked image with histogram distributions import seaborn as sns len(masked_img_s1.data) sns.distplot(masked_img_s1.data) # %% read in neurite from scan2 and generate histogram for comparison img_s2 = Brain_Data(neurite2) masked_img_s2 = img_s2.apply_mask(r_mask) masked_img_s2.plot() masked_img_s2.plot(anatomical=anat) len(masked_img_s2.data) sns.distplot(masked_img_s2.data) # ==== Test ground for playing ==== # # %% plotting glass brain /w `nilearn` import nilearn from nilearn import plotting plotting.plot_glass_brain(neurite1) # %% testing smoothing on the image from nilearn import image
roi_names = ['V1', 'A1', 'Precentral gyrus'] roi_ids = [int(rois.query(f'Region == "{r}"')['ID']) for r in roi_names] my_rois = {k:v for k, v in zip(roi_names, roi_ids)} # load subject's data and extract roi data2 = {} for run in ['Part1', 'Part2']: file_list = lsdir(os.path.join(datadir, 'fmriprep', '*', 'func', f'*crop*{run}*hdf5')) all_sub_roi = {} for f_name in file_list: sub_dat = Brain_Data(f_name) sub = os.path.basename(f_name).split('_')[0] print(sub, run) sub_roi = {} for roi in my_rois: sub_roi[roi] = [sub_dat.apply_mask(vectorized_mask[my_rois[roi]])] all_sub_roi[sub] = sub_roi data2[run] = all_sub_roi # rearrange data into new dictionary data = {} for run in data2.keys(): sub_list = list(data2[run].keys()) sub_list.sort() roi_dat = {} for roi in my_rois: sub_roi = [] for sub in sub_list: sub_roi.append(data2[run][sub][roi][0]) roi_dat[roi] = sub_roi data[run] = roi_dat
def test_brain_data(tmpdir): sim = Simulator() r = 10 sigma = 1 y = [0, 1] n_reps = 3 output_dir = str(tmpdir) sim.create_data(y, sigma, reps=n_reps, output_dir=output_dir) shape_3d = (91, 109, 91) shape_2d = (6, 238955) y=pd.read_csv(os.path.join(str(tmpdir.join('y.csv'))), header=None,index_col=None).T holdout=pd.read_csv(os.path.join(str(tmpdir.join('rep_id.csv'))),header=None,index_col=None).T flist = glob.glob(str(tmpdir.join('centered*.nii.gz'))) # Test load list dat = Brain_Data(data=flist,Y=y) # Test load file assert Brain_Data(flist[0]) # Test to_nifti d = dat.to_nifti() assert d.shape[0:3] == shape_3d # Test load nibabel assert Brain_Data(d) # Test shape assert dat.shape() == shape_2d # Test Mean assert dat.mean().shape()[0] == shape_2d[1] # Test Std assert dat.std().shape()[0] == shape_2d[1] # Test add new = dat + dat assert new.shape() == shape_2d # Test subtract new = dat - dat assert new.shape() == shape_2d # Test multiply new = dat * dat assert new.shape() == shape_2d # Test Iterator x = [x for x in dat] assert len(x) == len(dat) assert len(x[0].data.shape) == 1 # # Test T-test out = dat.ttest() assert out['t'].shape()[0] == shape_2d[1] # # # Test T-test - permutation method # out = dat.ttest(threshold_dict={'permutation':'tfce','n_permutations':50,'n_jobs':1}) # assert out['t'].shape()[0]==shape_2d[1] # Test Regress dat.X = pd.DataFrame({'Intercept':np.ones(len(dat.Y)), 'X1':np.array(dat.Y).flatten()},index=None) out = dat.regress() assert out['beta'].shape() == (2,shape_2d[1]) # Test indexing assert out['t'][1].shape()[0] == shape_2d[1] # Test threshold i=1 tt = threshold(out['t'][i], out['p'][i], .05) assert isinstance(tt,Brain_Data) # Test write dat.write(os.path.join(str(tmpdir.join('test_write.nii')))) assert Brain_Data(os.path.join(str(tmpdir.join('test_write.nii')))) # Test append assert dat.append(dat).shape()[0]==shape_2d[0]*2 # Test distance distance = dat.distance(method='euclidean') assert isinstance(distance,Adjacency) assert distance.square_shape()[0]==shape_2d[0] # Test predict stats = dat.predict(algorithm='svm', cv_dict={'type': 'kfolds','n_folds': 2}, plot=False,**{'kernel':"linear"}) # Support Vector Regression, with 5 fold cross-validation with Platt Scaling # This will output probabilities of each class stats = dat.predict(algorithm='svm', cv_dict=None, plot=False,**{'kernel':'linear', 'probability':True}) assert isinstance(stats['weight_map'],Brain_Data) # Logistic classificiation, with 2 fold cross-validation. stats = dat.predict(algorithm='logistic', cv_dict={'type': 'kfolds', 'n_folds': 2}, plot=False) assert isinstance(stats['weight_map'],Brain_Data) # Ridge classificiation, stats = dat.predict(algorithm='ridgeClassifier', cv_dict=None,plot=False) assert isinstance(stats['weight_map'],Brain_Data) # Ridge stats = dat.predict(algorithm='ridge', cv_dict={'type': 'kfolds', 'n_folds': 2,'subject_id':holdout}, plot=False,**{'alpha':.1}) # Lasso stats = dat.predict(algorithm='lasso', cv_dict={'type': 'kfolds', 'n_folds': 2,'stratified':dat.Y}, plot=False,**{'alpha':.1}) # PCR stats = dat.predict(algorithm='pcr', cv_dict=None, plot=False) # Test Similarity r = dat.similarity(stats['weight_map']) assert len(r) == shape_2d[0] r2 = dat.similarity(stats['weight_map'].to_nifti()) assert len(r2) == shape_2d[0] # Test apply_mask - might move part of this to test mask suite s1 = create_sphere([12, 10, -8], radius=10) assert isinstance(s1, nb.Nifti1Image) s2 = Brain_Data(s1) masked_dat = dat.apply_mask(s1) assert masked_dat.shape()[1] == np.sum(s2.data != 0) # Test extract_roi mask = create_sphere([12, 10, -8], radius=10) assert len(dat.extract_roi(mask)) == shape_2d[0] # Test r_to_z z = dat.r_to_z() assert z.shape() == dat.shape() # Test copy d_copy = dat.copy() assert d_copy.shape() == dat.shape() # Test detrend detrend = dat.detrend() assert detrend.shape() == dat.shape() # Test standardize s = dat.standardize() assert s.shape() == dat.shape() assert np.isclose(np.sum(s.mean().data), 0, atol=.1) s = dat.standardize(method='zscore') assert s.shape() == dat.shape() assert np.isclose(np.sum(s.mean().data), 0, atol=.1) # Test Sum s = dat.sum() assert s.shape() == dat[1].shape() # Test Groupby s1 = create_sphere([12, 10, -8], radius=10) s2 = create_sphere([22, -2, -22], radius=10) mask = Brain_Data([s1, s2]) d = dat.groupby(mask) assert isinstance(d, Groupby) # Test Aggregate mn = dat.aggregate(mask, 'mean') assert isinstance(mn, Brain_Data) assert len(mn.shape()) == 1 # Test Threshold s1 = create_sphere([12, 10, -8], radius=10) s2 = create_sphere([22, -2, -22], radius=10) mask = Brain_Data(s1)*5 mask = mask + Brain_Data(s2) m1 = mask.threshold(thresh=.5) m2 = mask.threshold(thresh=3) m3 = mask.threshold(thresh='98%') assert np.sum(m1.data > 0) > np.sum(m2.data > 0) assert np.sum(m1.data > 0) == np.sum(m3.data > 0) # Test Regions r = mask.regions(min_region_size=10) m1 = Brain_Data(s1) m2 = r.threshold(1, binarize=True) # assert len(r)==2 assert len(np.unique(r.to_nifti().get_data())) == 2 # JC edit: I think this is what you were trying to do diff = m2-m1 assert np.sum(diff.data) == 0
def train_model(image_list, algorithm, cross_validation, output_dir, file_path_key='resampled_file', mask=None): """ :param image_list: A list of dictionaries of the form { 'collection_id': '504', 'filename': 'Pain_Subject_1_Low.nii.gz', 'target': '1', 'resampled_file': 'path/to/the/resampled/file.nii.gz', 'original_file': 'path/to/the/original/file.nii.gz' } """ tic = time.time() # Start Timer try: holdout = [int(item['subject_id']) for item in image_list] except KeyError: holdout = None if cross_validation: if holdout: cross_validation['subject_id'] = holdout elif cross_validation['type'] == 'loso': raise ValueError( "subject_id is required for a LOSO cross validation.") extra = {} if algorithm in ('svr', 'svm'): extra = {'kernel': 'linear'} categorical_mapping = None if algorithm in CLASSIFICATION_ALGORITHMS: classes = {item['target'] for item in image_list} assert len(classes) == 2, ('More than two classes. ' 'Classification requires binary data.') categorical_mapping = {cls: index for index, cls in enumerate(classes)} for image in image_list: image['target'] = categorical_mapping[image['target']] Y = pd.DataFrame([float(item['target']) for item in image_list]) file_path_list = [item[file_path_key] for item in image_list] dat = Brain_Data(data=file_path_list, Y=Y) if mask: log.info('Applying a mask') nifti_mask = nb.load(mask) dat = dat.apply_mask(nifti_mask) output = dat.predict(algorithm=algorithm, cv_dict=cross_validation, plot=True, **extra) weightmap_filename = '%s_weightmap.nii.gz' % algorithm output['weight_map'].write(os.path.join(output_dir, weightmap_filename)) log.info("Elapsed: %.2f seconds", (time.time() - tic)) # Stop timer result = {'weightmap': weightmap_filename, 'intercept': float(output['intercept']), 'scatterplot': '%s_scatterplot.png ' % algorithm, 'stats': {key: output[key].tolist() for key in ('Y', 'yfit_xval', 'yfit_all') if key in output}, 'categorical_mapping': categorical_mapping, 'summary': get_summary(output)} if 'roc' in output: result['roc'] = output['roc'] return result
#call("3dcalc -a masks[0] -b masks[1] -c masks[2] -d masks[3] -expr 'ispositive((a+b+c+d)-0)' " # " -prefix {0}_brainmask.nii.gz".format(sub), shell=True) smoothed_fns = sorted( glob.glob( join(base_dir, 'derivatives', 'fmriprep', sub, ses, 'func', '*_4mm.nii.gz'))) wb_mask = join(base_dir, 'derivatives', 'fmriprep', sub, ses, 'func', '{0}_brainmask.nii.gz').format(sub) alldat = Brain_Data() for fn in smoothed_fns: dat = Brain_Data(fn) alldat = alldat.append(dat) maskeddat = alldat.apply_mask(nib.load(wb_mask)) mn = np.mean(maskeddat.data, axis=0) sd = np.std(maskeddat.data, axis=0) tsnr = np.true_divide(mn, sd) # Compute mean across voxels within each TR global_mn = np.mean(maskeddat.data, axis=1) global_sd = np.std(maskeddat.data, axis=1) # Unmask data for plotting below mn = unmask(mn, wb_mask) sd = unmask(sd, wb_mask) tsnr = unmask(tsnr, wb_mask) # Identify global signal outliers global_outliers = np.append( np.where(global_mn > np.mean(global_mn) + np.std(global_mn) * 3), np.where(global_mn < np.mean(global_mn) - np.std(global_mn) * 3))