def generate_analysis_images(self,
image_dir=None,
analyses=None,
add_to_db=True,
overwrite=True,
**kwargs):
""" Create a full set of analysis meta-analysis images via Neurosynth.
Args:
image_dir: Folder in which to store images. If None, uses default
location specified in SETTINGS.
analyses: Optional list of analyses to limit meta-analysis to.
If None, all available analyses are processed.
add_to_db: if True, will create new AnalysisImage records, and
associate them with the corresponding Analysis record.
overwrite: if True, always generate new meta-analysis images. If
False, will skip any analyses that already have images.
kwargs: optional keyword arguments to pass onto the Neurosynth
meta-analysis.
"""
# Set up defaults
if image_dir is None:
image_dir = join(settings.IMAGE_DIR, 'analyses')
if not os.path.exists(image_dir):
os.makedirs(image_dir)
os.makedirs(join(settings.IMAGE_DIR, 'custom'))
if analyses is None:
analyses = self._get_feature_names()
# Remove analyses that already exist
if not overwrite:
files = glob(
join(settings.IMAGE_DIR, 'analyses',
'*_association-test_z.nii.gz'))
existing = [basename(f).split('_')[0] for f in files]
analyses = list(set(analyses) - set(existing))
# Meta-analyze all images
meta.analyze_features(self.dataset,
analyses,
output_dir=image_dir,
q=0.01,
**kwargs)
# Create AnalysisImage records
if add_to_db:
for f in analyses:
self.add_analysis_images(f, image_dir)
self.db.session.commit()
def test_meta_analysis(self):
""" Test full meta-analysis stream. """
# run a meta-analysis
ids = ['study1', 'study3']
ma = meta.MetaAnalysis(self.dataset, ids)
# save the results
tempdir = tempfile.mkdtemp()
ma.save_results(tempdir + os.path.sep, prefix='test')
files = glob(tempdir + os.path.sep + "test_*.nii.gz")
self.assertEquals(len(files), 9)
shutil.rmtree(tempdir)
# test the analyze_features() wrapper
tempdir = tempfile.mkdtemp()
meta.analyze_features(self.dataset, output_dir=tempdir, prefix="meep")
files = glob(tempdir + os.path.sep + "meep*.nii.gz")
self.assertEquals(len(files), 9 * 5)
shutil.rmtree(tempdir)
def generate_analysis_images(self, image_dir=None, analyses=None,
add_to_db=True, overwrite=True, **kwargs):
"""
Create a full set of analysis meta-analysis images via Neurosynth.
Args:
image_dir: Folder in which to store images. If None, uses default
location specified in SETTINGS.
analyses: Optional list of analyses to limit meta-analysis to.
If None, all available analyses are processed.
add_to_db: if True, will create new AnalysisImage records, and
associate them with the corresponding Analysis record.
overwrite: if True, always generate new meta-analysis images. If
False, will skip any analyses that already have images.
kwargs: optional keyword arguments to pass onto the Neurosynth
meta-analysis.
"""
# Set up defaults
if image_dir is None:
image_dir = join(settings.IMAGE_DIR, 'analyses')
if not os.path.exists(image_dir):
os.makedirs(image_dir)
os.makedirs(join(settings.IMAGE_DIR, 'custom'))
if analyses is None:
analyses = self._get_feature_names()
# Remove analyses that already exist
if not overwrite:
files = glob(
join(settings.IMAGE_DIR, 'analyses', '*_pFgA_z.nii.gz'))
existing = [basename(f).split('_')[0] for f in files]
analyses = list(set(analyses) - set(existing))
# Meta-analyze all images
meta.analyze_features(self.dataset, analyses, output_dir=image_dir,
q=0.01, **kwargs)
# Create AnalysisImage records
if add_to_db:
for f in analyses:
self.add_analysis_images(f, image_dir)
self.db.session.commit()
def test_meta_analysis(self):
""" Test full meta-analysis stream. """
# run a meta-analysis
ids = ['study1', 'study3']
ma = meta.MetaAnalysis(self.dataset, ids)
# save the results
tempdir = tempfile.mkdtemp()
ma.save_results(tempdir + os.path.sep, prefix='test')
files = glob(tempdir + os.path.sep + "test_*.nii.gz")
self.assertEquals(len(files), 9)
shutil.rmtree(tempdir)
# test the analyze_features() wrapper
tempdir = tempfile.mkdtemp()
meta.analyze_features(
self.dataset, output_dir=tempdir, prefix="meep")
files = glob(tempdir + os.path.sep + "meep*.nii.gz")
self.assertEquals(len(files), 9*5)
shutil.rmtree(tempdir)
def _load_features_from_dataset(self, features=None, image_type=None):
""" Load feature image data from the current Dataset instance.
Args:
features: Optional list of features to use. If None, all features in the
current Dataset instance will be used.
"""
self.feature_names = self.dataset.feature_table.feature_names
if features is not None:
self.feature_names = filter(lambda x: x in self.feature_names, features)
from neurosynth.analysis import meta
self.feature_images = meta.analyze_features(self.dataset, self.feature_names, image_type=image_type)
def _load_features_from_dataset(self, features=None, image_type=None, threshold=0.001):
""" Load feature image data from the current Dataset instance. See load_features()
for documentation.
"""
self.feature_names = self.dataset.feature_table.feature_names
if features is not None:
self.feature_names = filter(lambda x: x in self.feature_names, features)
from neurosynth.analysis import meta
self.feature_images = meta.analyze_features(
self.dataset, self.feature_names, image_type=image_type, threshold=threshold)
# Apply a mask if one was originally passed
if self.masker.layers:
in_mask = self.masker.get_current_mask(in_global_mask=True)
self.feature_images = self.feature_images[in_mask,:]
def _load_features_from_dataset(self, features=None, image_type=None):
""" Load feature image data from the current Dataset instance.
Args:
features: Optional list of features to use. If None, all features in the
current Dataset instance will be used.
"""
self.feature_names = self.dataset.feature_table.feature_names
if features is not None:
self.feature_names = filter(lambda x: x in self.feature_names,
features)
from neurosynth.analysis import meta
self.feature_images = meta.analyze_features(self.dataset,
self.feature_names,
image_type=image_type)
def _load_features_from_dataset(self, features=None, image_type=None,
threshold=0.001):
""" Load feature image data from the current Dataset instance. See
load_features() for documentation.
"""
self.feature_names = self.dataset.feature_table.feature_names
if features is not None:
self.feature_names = [f for f in features if f in self.feature_names]
from neurosynth.analysis import meta
self.feature_images = meta.analyze_features(
self.dataset, self.feature_names, image_type=image_type,
threshold=threshold)
# Apply a mask if one was originally passed
if self.masker.layers:
in_mask = self.masker.get_mask(in_global_mask=True)
self.feature_images = self.feature_images[in_mask, :]
def _load_features_from_dataset(self,
features=None,
image_type=None,
threshold=0.001):
""" Load feature image data from the current Dataset instance. See
load_features() for documentation.
"""
self.feature_names = self.dataset.feature_table.feature_names
if features is not None:
self.feature_names = [
f for f in features if f in self.feature_names
]
from neurosynth.analysis import meta
self.feature_images = meta.analyze_features(self.dataset,
self.feature_names,
image_type=image_type,
threshold=threshold)
from neurosynth.base.dataset import Dataset
from neurosynth.analysis import meta
""" Load a Dataset and generate a full set of meta-analysis
images--i.e., run a meta-analysis on every single feature.
"""
# Load pickled Dataset--assumes you've previously saved it. If not,
# follow the create_a_new_dataset_and_load_features example.
dataset = Dataset.load('dataset.pkl')
# Get the full list of feature names
feature_list = dataset.get_feature_names()
# Run a meta-analysis on each feature, and save all the results to
# a directory called results. Note that the directory will not be
# created for you, so make sure it exists.
# Here we use the default frequency threshold of 0.001 (i.e., a
# study is said to have a feature if more than 1 in every 1,000
# words is the target word), and an FDR correction level of 0.05.
meta.analyze_features(dataset,
feature_list,
threshold=0.001,
q=0.05,
save='results/')
from neurosynth import Dataset
from neurosynth.analysis import meta
from neurosynth.analysis import decode
neurosynth_data = "%s/neurosynth-data" % base
# you will need to open the features file in vim and add pmid as the first column name to appear before "action."
dataset = Dataset('%s/database.txt' % neurosynth_data, features_file)
# Generate maps for the features
neurosynth_feature_maps = "%s/feature_maps" % neurosynth_data
if not os.path.exists(neurosynth_feature_maps):
os.mkdir(neurosynth_feature_maps)
meta.analyze_features(dataset,
output_dir=neurosynth_feature_maps,
prefix='cog_atlas')
#...and boom, you should have a full set of images. (-Tal Yarkoni) :)
# we need to put image data into 2mm mask to resize to 4mm
df = pandas.DataFrame(dataset.image_table.data.toarray())
df.columns = features.index
# Make 4mm images
brain_4mm = get_standard_mask(4)
for pmid in df.columns:
pmid_mr = df[pmid].tolist()
empty_nii = numpy.zeros(dataset.masker.volume.shape)
empty_nii[dataset.masker.volume.get_data() != 0] = pmid_mr
empty_nii = nibabel.Nifti1Image(empty_nii,
def add_topics(self, generate_images=True, add_images=True, top_n=20,
reset=False):
""" Seed the database with topics.
Args:
generate_images (bool): if True, generates meta-analysis images for
all topics.
add_images (bool): if True, adds records for all images to the
database.
top_n: number of top-loading words to save.
reset: if True, drops all existing TopicSets and TopicAnalysis
records before repopulating.
"""
if reset:
for ts in AnalysisSet.query.filter_by(type='topics').all():
self.db.session.delete(ts)
for t in TopicAnalysis.query.all():
self.db.session.delete(t)
self.db.session.commit()
topic_sets = glob(join(settings.TOPIC_DIR, '*.json'))
# Temporarily store the existing AnalysisTable so we don't overwrite it
# feature_table = deepcopy(self.dataset.feature_table)
topic_image_dir = join(settings.IMAGE_DIR, 'topics')
# Get all valid Study ids for speed
study_ids = [x[0] for x in set(self.db.session.query(Study.pmid).all())]
for ts in topic_sets:
data = json.load(open(ts))
n = int(data['n_topics'])
topic_set_image_dir = join(topic_image_dir, data['name'])
if not exists(topic_set_image_dir):
os.makedirs(topic_set_image_dir)
self.dataset.add_features(
join(settings.TOPIC_DIR, 'analyses',
data['name'] + '.txt'), append=False)
key_data = open(join(settings.TOPIC_DIR, 'keys', data['name'] +
'.txt')).read().splitlines()
# Generate full set of topic images
if generate_images:
meta.analyze_features(
self.dataset, self.dataset.get_feature_names(),
output_dir=topic_set_image_dir, threshold=0.05, q=0.01,
prefix=data['name'])
ts = AnalysisSet(name=data['name'],
description=data['description'], n_analyses=n,
type='topics')
self.db.session.add(ts)
feature_data = self.dataset.feature_table.data
feature_names = self.dataset.get_feature_names()
for i, fn in enumerate(feature_names):
number = int(fn.split('_')[0])
name = '%s_%s' % (ts.name, fn)
# Disable autoflush temporarily because it causes problems
with self.db.session.no_autoflush:
terms = ', '.join(key_data[number].split()[2:][:top_n])
# Topics are not always in natsort order; get correct number
topic = TopicAnalysis(name=name, terms=terms, number=number)
self.db.session.add(topic)
self.db.session.commit()
# Map onto studies
freqs = feature_data[fn]
above_thresh = freqs[freqs >= 0.05]
srsly = 0
for s_id, f in above_thresh.iteritems():
s_id = int(s_id)
if s_id in study_ids:
srsly += 1
self.db.session.add(Frequency(
pmid=s_id, analysis=topic, frequency=f))
# Update counts
topic.n_studies = len(above_thresh)
self.db.session.commit()
if add_images:
self.add_analysis_images(topic, topic_set_image_dir)
ts.analyses.append(topic)
self.db.session.commit()
from neurosynth.base.dataset import Dataset
from neurosynth.analysis import meta
""" Load a Dataset and generate a full set of meta-analysis
images--i.e., run a meta-analysis on every single feature.
"""
# Load pickled Dataset--assumes you've previously saved it. If not,
# follow the create_a_new_dataset_and_load_features example.
dataset = Dataset.load('dataset.pkl')
# Get the full list of feature names
feature_list = dataset.get_feature_names()
# Run a meta-analysis on each feature, and save all the results to
# a directory called results. Note that the directory will not be
# created for you, so make sure it exists.
# Here we use the default frequency threshold of 0.001 (i.e., a
# study is said to have a feature if more than 1 in every 1,000
# words is the target word), and an FDR correction level of 0.05.
meta.analyze_features(dataset, feature_list, threshold=0.001, q=0.05, save='results/')
def add_topics(self, generate_images=True, add_images=True, top_n=20,
reset=False):
""" Seed the database with topics.
Args:
generate_images (bool): if True, generates meta-analysis images for
all topics.
add_images (bool): if True, adds records for all images to the
database.
top_n: number of top-loading words to save.
reset: if True, drops all existing TopicSets and TopicAnalysis
records before repopulating.
"""
if reset:
for ts in AnalysisSet.query.filter_by(type='topics').all():
self.db.session.delete(ts)
for t in TopicAnalysis.query.all():
self.db.session.delete(t)
self.db.session.commit()
topic_sets = glob(join(settings.TOPIC_DIR, '*.json'))
# Temporarily store the existing AnalysisTable so we don't overwrite it
# feature_table = deepcopy(self.dataset.feature_table)
topic_image_dir = join(settings.IMAGE_DIR, 'topics')
# Get all valid Study ids for speed
study_ids = [x[0] for x in set(self.db.session.query(Study.pmid).all())]
for ts in topic_sets:
data = json.load(open(ts))
n = int(data['n_topics'])
topic_set_image_dir = join(topic_image_dir, data['name'])
if not exists(topic_set_image_dir):
os.makedirs(topic_set_image_dir)
self.dataset.add_features(
join(settings.TOPIC_DIR, 'analyses',
data['name'] + '.txt'), append=False)
key_data = open(join(settings.TOPIC_DIR, 'keys', data['name'] +
'.txt')).read().splitlines()
# Generate full set of topic images
if generate_images:
meta.analyze_features(
self.dataset, self.dataset.get_feature_names(),
output_dir=topic_set_image_dir, threshold=0.05, q=0.01,
prefix=data['name'])
ts = AnalysisSet(name=data['name'],
description=data['description'], n_analyses=n,
type='topics')
self.db.session.add(ts)
feature_data = self.dataset.feature_table.data
feature_names = self.dataset.get_feature_names()
for i, fn in enumerate(feature_names):
number = int(fn.split('_')[0])
name = '%s_%s' % (ts.name, fn)
# Disable autoflush temporarily because it causes problems
with self.db.session.no_autoflush:
terms = ', '.join(key_data[number].split()[2:][:top_n])
# Topics are not always in natsort order; get correct number
topic = TopicAnalysis(name=name, terms=terms, number=number)
self.db.session.add(topic)
self.db.session.commit()
# Map onto studies
freqs = feature_data[fn]
above_thresh = freqs[freqs >= 0.05]
srsly = 0
for s_id, f in above_thresh.iteritems():
s_id = int(s_id)
if s_id in study_ids:
srsly += 1
self.db.session.add(Frequency(
pmid=s_id, analysis=topic, frequency=f))
# Update counts
topic.n_studies = len(above_thresh)
self.db.session.commit()
if add_images:
self.add_analysis_images(topic, topic_set_image_dir)
ts.analyses.append(topic)
self.db.session.commit()
def do_full_analysis(dataset, image_directory): feature_list = dataset.get_feature_names() meta.analyze_features(dataset, feature_list, threshold=0.001,save=image_directory)
def do_full_analysis(dataset): feature_list = dataset.get_feature_names() meta.analyze_features(dataset, feature_list, threshold=0.001,save='results-old/')
# Pickle the Dataset to file so we can use Dataset.load() next time
# instead of having to sit through the generation process again.
dataset.save(op.join(neurosynth_data_dir, "dataset.pkl"))
# Load pickled Dataset--assumes you've previously saved it. If not,
# follow the create_a_new_dataset_and_load_features example.
dataset = Dataset.load(op.join(neurosynth_data_dir, "dataset.pkl"))
# Get the full list of feature names
feature_list = dataset.get_feature_names()
# Run a meta-analysis on each feature, and save all the results to
# a directory called results. Note that the directory will not be
# created for you, so make sure it exists.
# Here we use the default frequency threshold of 0.001 (i.e., a
# study is said to have a feature if more than 1 in every 1,000
# words is the target word), and an FDR correction level of 0.05.
out_dir = "/home/data/nbc/misc-projects/meta-gradients/code/feature_maps"
for tmp_feature in feature_list:
print(tmp_feature)
meta.analyze_features(
dataset,
[tmp_feature],
threshold=0.001,
image_type="association-test_z",
output_dir=out_dir,
q=0.01,
)
from neurosynth import Dataset
from neurosynth.analysis import meta
from neurosynth.analysis import decode
neurosynth_data = "%s/neurosynth-data" %base
# you will need to open the features file in vim and add pmid as the first column name to appear before "action."
dataset = Dataset('%s/database.txt' %neurosynth_data,features_file)
# Generate maps for the features
neurosynth_feature_maps = "%s/feature_maps" %neurosynth_data
if not os.path.exists(neurosynth_feature_maps):
os.mkdir(neurosynth_feature_maps)
meta.analyze_features(dataset, output_dir=neurosynth_feature_maps, prefix='cog_atlas')
#...and boom, you should have a full set of images. (-Tal Yarkoni) :)
# we need to put image data into 2mm mask to resize to 4mm
df = pandas.DataFrame(dataset.image_table.data.toarray())
df.columns = features.index
# Make 4mm images
brain_4mm = get_standard_mask(4)
for pmid in df.columns:
pmid_mr = df[pmid].tolist()
empty_nii = numpy.zeros(dataset.masker.volume.shape)
empty_nii[dataset.masker.volume.get_data()!=0] = pmid_mr
empty_nii = nibabel.Nifti1Image(empty_nii,affine=dataset.masker.volume.get_affine())
tmpnii = "%s/tmp.nii.gz" %(neurosynth_feature_maps)