def get_data(download_folder): """get_data :param download_folder: download destination folder for nifti gz files :returns brainmaps: list of nii gz images """ all_collections = api.get_collections() collections_with_dois = all_collections[np.logical_not(all_collections.DOI.isnull())] group_collections = collections_with_dois[collections_with_dois.group_comparison != True] images = api.get_images(collection_pks=list(group_collections.collection_id)) images = api.filter(df=images, column_name="is_thresholded", field_value=False) images = api.filter(df=images, column_name="not_mni", field_value=False) images = api.filter(df=images, column_name="modality", field_value="fMRI-BOLD") api.download_images(dest_dir=download_folder, images_df=images, resample=False) return glob("%s/original/*.nii.gz" % (download_folder))
def get_data(download_folder): '''get_data :param download_folder: download destination folder for nifti gz files :returns brainmaps: list of nii gz images ''' all_collections = api.get_collections() collections_with_dois = all_collections[np.logical_not( all_collections.DOI.isnull())] group_collections = collections_with_dois[ collections_with_dois.group_comparison != True] images = api.get_images( collection_pks=list(group_collections.collection_id)) images = api.filter(df=images, column_name="is_thresholded", field_value=False) images = api.filter(df=images, column_name="not_mni", field_value=False) images = api.filter(df=images, column_name="modality", field_value="fMRI-BOLD") api.download_images(dest_dir=download_folder, images_df=images, resample=False) return glob("%s/original/*.nii.gz" % (download_folder))
images.cognitive_paradigm_cogatlas.isin(["None / Other", "rest eyes closed", "rest eyes open"]) == False ] # Limit to Z and T maps (all are Z and T) z = images[images.map_type == "Z map"] t = images[images.map_type == "T map"] # Remove tmaps that do not have # subjects defined t_collections = collections[collections.collection_id.isin([int(x) for x in t.collection_id])] to_keep = t_collections.collection_id[t_collections.number_of_subjects.isnull() == False] t = t[t.collection_id.isin(to_keep)] images = z.append(t) # Download images standard = os.path.abspath("mr/MNI152_T1_2mm_brain.nii.gz") api.download_images(dest_dir=data, images_df=images, target=standard) # For T images, convert to Z tmaps = ["%s/resampled/%06d.nii.gz" % (data, x) for x in t.image_id.tolist()] # Look up number of subjects, and calculate dofs dofs = [] for row in t.iterrows(): dof = collections.number_of_subjects[collections.collection_id == int(row[1].collection_id)].tolist()[0] - 2 dofs.append(dof) outfolder_z = "%s/resampled_z" % (data) if not os.path.exists(outfolder_z): os.mkdir(outfolder_z) for tt in range(0, len(tmaps)):
from pyneurovault import api import numpy as np all_collections = api.get_collections() collections_with_dois = all_collections[np.logical_not(all_collections.DOI.isnull())] images = api.get_images(collection_pks=list(collections_with_dois.collection_id)) # Remove images that are thresholded images = api.filter(df=images,column_name="is_thresholded",field_value=False) # Not in MNI images = api.filter(df=images,column_name="not_mni",field_value=False) # Just fMRI bold images = api.filter(df=images,column_name="modality",field_value="fMRI-BOLD") # Download all images to file, resample to target outfolder = "/home/vanessa/Documents/Work/NEUROVAULT/mosaic" # If you don't want to resample api.download_images(dest_dir = outfolder,images_df=images,resample=False)
from pyneurovault import api import pandas as pd from nilearn.image import resample_img, smooth_img import nibabel as nb import numpy as np from scipy.ndimage import binary_fill_holes from scipy import stats from scipy.ndimage.measurements import labeled_comprehension import sys atlas_id = sys.argv[2] data_location = sys.argv[1] atlases = api.get_images(pks=[atlas_id]) api.download_images(data_location + "/atlases", atlases, resample=False) standard = "/usr/share/fsl/data/standard/MNI152_T1_2mm_brain.nii.gz" get_ipython().system('mkdir -p ' + data_location + '/atlases/resampled') fname = atlas_id + ".nii.gz" print "resampling " + fname nii = nb.load(data_location + "atlases/original/" + fname) target_nii = nb.load(standard) resampled_nii = resample_img(nii, target_affine=target_nii.get_affine(), target_shape=target_nii.shape, interpolation='nearest') resampled_nii.to_filename(data_location + "atlases/resampled/" + fname)
# We can't use Rest or other/none images = images[images.cognitive_paradigm_cogatlas_id.isnull() == False] images = images[images.cognitive_paradigm_cogatlas.isin( ["None / Other", "rest eyes closed", "rest eyes open"]) == False] # Limit to Z and T maps (all are Z and T) z = images[images.map_type == "Z map"] t = images[images.map_type == "T map"] # Remove tmaps that do not have # subjects defined t = t[t.number_of_subjects.isnull() == False] images = z.append(t) # Download images standard = os.path.abspath("%s/mr/MNI152_T1_2mm_brain.nii.gz" % (here)) api.download_images(dest_dir=data_directory, images_df=images, target=standard) # For T images, convert to Z. NeuroVault outputs two folders - original and resampled resampled_dir = "%s/resampled" % (data_directory) # We need to select a subset of the images, just the T Maps from the set tmaps = ["%s/%06d.nii.gz" % (resampled_dir, x) for x in t.image_id.tolist()] # We need degrees of freedom to convert properly to Zstat maps. dofs = [] for row in t.iterrows(): dof = row[1].number_of_subjects - 2 dofs.append(dof) # We will move converted Z maps, and as is Z maps, to a common folder outfolder_z = "%s/resampled_z" % (data_directory)
def download_combined_database(out_dir, overwrite=False): """ Download coordinates/annotations from brainspell and images/annotations from Neurovault. Currently, the largest barrier is the lack of links between experiments (tables) in brainspell/NeuroSynth and those in NeuroVault. The closest we have is overall papers, via DOIs. Additional problems: - Does NeuroVault have standard error maps? - If so, I doubt there's any way to associate a given SE map and beta map within a collection. - How should space be handled? - Should everything be resliced and transformed to the same space at this stage or later on? - How can we link a target template (for images) to a target space (for coordinates)? - Should we even allow custom targets? Maybe we just limit it to 2mm and 1mm MNI templates. Parameters ---------- out_dir : :obj:`str` Folder in which to write out Dataset object and subfolders containing images. overwrite: :obj:`bool`, optional Whether to overwrite existing database, if one exists in `out_dir`. Defaults to False. """ # Download collections metadata from Neurovault collections_file = op.join(out_dir, 'neurovault_collections.csv') if overwrite or not op.isfile(collections_file): colls_df = api.get_collections() colls_df.to_csv(collections_file, index=False, encoding='utf-8') else: colls_df = pd.read_csv(collections_file, encoding='utf-8') # Only include collections from published papers (or preprints) papers_file = op.join(out_dir, 'neurovault_papers.csv') if overwrite or not op.isfile(papers_file): paper_df = colls_df.dropna(subset=['DOI']) paper_df.to_csv(papers_file, index=False, encoding='utf-8') else: paper_df = pd.read_csv(papers_file, encoding='utf-8') # Get metadata for individual images from valid collections papers_metadata_file = op.join(out_dir, 'neurovault_papers_metadata.csv') if overwrite or not op.isfile(papers_metadata_file): valid_collections = sorted(paper_df['collection_id'].tolist()) # Sleep between get_images calls to avoid spamming Neurovault image_dfs = [] for chunk in to_chunks(valid_collections, 500): image_dfs.append(api.get_images(collection_pks=chunk)) time.sleep(10) image_df = pd.concat(image_dfs) image_df.to_csv(papers_metadata_file, index=False, encoding='utf-8') else: image_df = pd.read_csv(papers_metadata_file, encoding='utf-8') # Reduce images database according to additional criteria # Only keep unthresholded, MNI, group level fMRI maps red_df = image_df.loc[image_df['modality'] == 'fMRI-BOLD'] red_df = red_df.loc[red_df['image_type'] == 'statistic_map'] red_df = red_df.loc[red_df['analysis_level'] == 'group'] red_df = red_df.loc[red_df['is_thresholded'] is False] red_df = red_df.loc[red_df['not_mni'] is False] # Look for relevant metadata red_df = red_df.dropna(subset=['cognitive_paradigm_cogatlas']) ## MFX/FFX GLMs need contrast (beta) + standard error mffx_df = red_df.loc[red_df['map_type'] == 'univariate-beta map'] ## RFX GLMs need contrast (beta) rfx_df = red_df.loc[red_df['map_type'] == 'univariate-beta map'] ## Stouffer's, Stouffer's RFX, and Fisher's IBMAs can use Z maps. # T and F maps can be transformed into Z maps, but T maps need sample size. # Only keep test statistic maps acc_map_types = ['Z map', 'T map', 'F map'] st_df = red_df.loc[red_df['map_type'].isin(acc_map_types)] keep_idx = st_df['map_type'].isin(['Z map', 'F map']) keep_idx2 = (st_df['map_type'] == 'T map') & ~pd.isnull(st_df['number_of_subjects']) keep_idx = keep_idx | keep_idx2 st_df = st_df.loc[keep_idx] ## Weighted Stouffer's IBMAs need Z + sample size. st_df['id_str'] = st_df['image_id'].astype(str).str.zfill(6) if not op.isdir(out_dir): mkdir(out_dir) api.download_images(out_dir, red_df, target=None, resample=False) elif overwrite: # clear out out_dir raise Exception('Currently not prepared to overwrite database.') api.download_images(out_dir, red_df, target=None, resample=False)
from pyneurovault import api import pandas as pd from nilearn.image import resample_img, smooth_img import nibabel as nb import numpy as np from scipy.ndimage import binary_fill_holes from scipy import stats from scipy.ndimage.measurements import labeled_comprehension import sys atlas_id = sys.argv[2] data_location = sys.argv[1] atlases = api.get_images(pks=[atlas_id]) api.download_images(data_location + "/atlases", atlases, resample=False) standard = "/usr/share/fsl/data/standard/MNI152_T1_2mm_brain.nii.gz" get_ipython().system('mkdir -p '+data_location+'/atlases/resampled') fname = atlas_id+".nii.gz" print "resampling " + fname nii = nb.load(data_location+"atlases/original/" + fname) target_nii = nb.load(standard) resampled_nii = resample_img(nii,target_affine=target_nii.get_affine(), target_shape=target_nii.shape, interpolation='nearest') resampled_nii.to_filename(data_location+"atlases/resampled/" + fname) def score_map(map_filename, atlas_data, labels): map_data = nb.load(map_filename).get_data()
# Get a collection collection = api.get_collections(pks=457) # collection.collection_id is 457 # Get all images images = api.get_images() # Get all images meta data for a collection images = api.get_images(collection_pks=457) # Remove images that are thresholded images = api.filter(df=images,column_name="is_thresholded",field_value=False) # Not in MNI images = api.filter(df=images,column_name="not_mni",field_value=False) # Just fMRI bold images = api.filter(df=images,column_name="modality",field_value="fMRI-BOLD") # Download images, collections, or both api.export_images_tsv("/home/vanessa/Desktop/images.tsv",images) api.export_collections_tsv("/home/vanessa/Desktop/collections.tsv",collection) # Download all images to file, resample to target outfolder = "/home/vanessa/Desktop" standard = "/usr/share/fsl/data/standard/MNI152_T1_2mm_brain.nii.gz" api.download_images(dest_dir = outfolder,images_df=images,target=standard) # If you don't want to resample api.download_images(dest_dir = outfolder,images_df=images,resample=False)