def main():
base = "data/"
# Make a folder for mean images
if not os.path.exists("mr"):
os.mkdir("mr")
# Get Neurovault Images with defined cognitive atlas contrast
collections = get_collections()
# Filter images to those that have a DOI
collections = collections[collections.DOI.isnull() == False]
# Get image meta data for collections (N=1023)
images = get_images(collection_pks=collections.collection_id.tolist())
# Filter images to those with contrasts defined (N=98)
images = images[images.cognitive_contrast_cogatlas_id.isnull() == False]
# Get rid of any not in MNI
images = images[images.not_mni == False]
# Get rid of thresholded images
images = images[images.is_thresholded == False]
### Step 1: Load meta data sources
unique_contrasts = images.cognitive_contrast_cogatlas_id.unique().tolist()
# Images that do not match the correct identifier will not be used (eg, "Other")
expression = re.compile("cnt_*")
unique_contrasts = [u for u in unique_contrasts if expression.match(u)]
# Make sure exists in cognitive atlas
existing_contrasts = []
for u in unique_contrasts:
try:
tmp = get_concept(contrast_id=u, silent=True)
existing_contrasts.append(u)
except:
print "%s is defined in NeuroVault, does not exist in Cognitive Atlas" % u
image_lookup = dict()
for u in existing_contrasts:
image_lookup[u] = images.image_id[images.cognitive_contrast_cogatlas_id
== u].tolist()
# Create a data structure of tasks and contrasts for our analysis
relationship_table = concept_node_triples(image_dict=image_lookup,
save_to_file=False)
unique_nodes = relationship_table.id.unique().tolist()
# We will store a data frame of meta data
# Lookup for meta_data is the id of the node!
meta_data = {}
for node in unique_nodes:
meta_single = {}
# This is an image node
if re.search("node_", node):
print "Found image node!"
relationship_table_row = relationship_table[relationship_table.id
== node]
image_id = relationship_table_row.name.tolist()[0]
meta_single["category"] = ""
meta_single["type"] = "nii"
# NeuroVault metadata
concepts = relationship_table.parent[relationship_table.name ==
image_id]
concepts = [
relationship_table.name[relationship_table.id == c].tolist()[0]
for c in concepts
]
neurovault_row = images[images.image_id == int(image_id)]
collection_row = collections[collections.collection_id ==
neurovault_row.collection_id.tolist()
[0]]
collection_meta = {
"DOI":
collection_row["DOI"].tolist()[0],
"authors":
collection_row["authors"].tolist()[0],
"journal":
collection_row["journal_name"].tolist()[0],
"url":
collection_row["url"].tolist()[0],
"subjects":
collection_row["number_of_subjects"].tolist()[0],
"smoothing_fwhm":
str(collection_row["smoothing_fwhm"].tolist()[0]).encode(
"utf-8")
}
meta_single["url"] = neurovault_row["url"].tolist()[0]
meta_single["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
meta_single["images"] = neurovault_row["thumbnail"].tolist()
meta_single["task"] = neurovault_row[
"cognitive_paradigm_cogatlas"].tolist()[0]
meta_single["contrast"] = neurovault_row[
"cognitive_contrast_cogatlas"].tolist()[0]
meta_single["download"] = neurovault_row["file"].tolist()[0]
meta_single["concept"] = concepts
if neurovault_row["description"].tolist()[0]:
meta_single["description"] = str(
neurovault_row["description"].tolist()[0]).encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single[
"description"][0:600]
else: # A concept node
if node != "1":
relationship_table_row = relationship_table[
relationship_table.id == node]
concept = get_concept(id=node, silent=True).json
children_nodes = [
relationship_table.name.tolist()[x]
for x in range(relationship_table.shape[0])
if relationship_table.parent.tolist()[x] == node
]
while len(
[x for x in children_nodes if not isinstance(x, int)]) > 0:
new_parent_nodes = [
x for x in children_nodes if not isinstance(x, int)
]
children_nodes = [
x for x in children_nodes if x not in new_parent_nodes
]
for new_parent in new_parent_nodes:
node_name = relationship_table.id[
relationship_table.name == new_parent].tolist()[0]
children_nodes = children_nodes + [
relationship_table.name.tolist()[x]
for x in range(relationship_table.shape[0]) if
relationship_table.parent.tolist()[x] == node_name
]
# Now only keep children that are images
meta_single["images"] = images["thumbnail"][
images.image_id.isin(children_nodes)].tolist()
# Cognitive Atlas meta data
meta_single[
"url"] = "http://www.cognitiveatlas.org/term/id/%s" % node
meta_single["type"] = "concept"
meta_single[
"thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
meta_single["concept"] = [
relationship_table.name[relationship_table.id ==
node].tolist()[0]
]
meta_single["task"] = ""
meta_single["contrast"] = []
meta_single[
"download"] = "http://www.cognitiveatlas.org/rdf/id/%s" % node
if concept[0]["definition_text"]:
meta_single["description"] = concept[0][
"definition_text"].encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single[
"description"][0:600]
meta_data[node] = meta_single
## STEP 2: VISUALIZATION WITH PYBRAINCOMPARE
from pybraincompare.ontology.tree import named_ontology_tree_from_tsv, make_ontology_tree_d3
# First let's look at the tree structure
# output_json = "%s/task_contrast_tree.json" % outfolder
tree = named_ontology_tree_from_tsv(relationship_table,
output_json=None,
meta_data=meta_data)
html_snippet = make_ontology_tree_d3(tree)
web_folder = base
make_analysis_web_folder(html_snippet, web_folder)
# To get a dump of just the tree (for use in more advanced custom web interface)
filey = open('%s/reverseinference.json' % base, 'wb')
filey.write(
json.dumps(tree, sort_keys=True, indent=4, separators=(',', ': ')))
filey.close()
## STEP 3: Export individual nodes
### Images
unique_images = images.image_id.unique().tolist()
# Images
for s in range(0, len(unique_images)):
image_id = unique_images[s]
meta_data = {}
meta_data["image_id"] = image_id
print "Parsing data for images %s of %s" % (s, len(unique_images))
concepts = relationship_table.parent[relationship_table.name == str(
image_id)].tolist()
concepts = [
relationship_table.name[relationship_table.id == c].tolist()[0]
for c in concepts
]
concepts_ids = [
relationship_table.id[relationship_table.id == c].tolist()[0]
for c in concepts
]
neurovault_row = images[images.image_id == int(image_id)]
collection_row = collections[collections.collection_id ==
neurovault_row.collection_id.tolist()[0]]
collection_meta = {
"DOI":
collection_row["DOI"].tolist()[0],
"authors":
collection_row["authors"].tolist()[0],
"journal":
collection_row["journal_name"].tolist()[0],
"url":
collection_row["url"].tolist()[0],
"subjects":
collection_row["number_of_subjects"].tolist()[0],
"smoothing_fwhm":
str(collection_row["smoothing_fwhm"].tolist()[0]).encode("utf-8"),
"title":
collection_row["name"].tolist()[0]
}
meta_data["collection"] = collection_meta
meta_data["url"] = neurovault_row["url"].tolist()[0]
meta_data["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
meta_data["images"] = neurovault_row["thumbnail"].tolist()
meta_data["task"] = neurovault_row[
"cognitive_paradigm_cogatlas"].tolist()[0]
meta_data["contrast"] = neurovault_row[
"cognitive_contrast_cogatlas"].tolist()[0]
meta_data["download"] = neurovault_row["file"].tolist()[0]
meta_data["concept"] = concepts
meta_data["concept_id"] = concepts_ids
if neurovault_row["description"].tolist()[0]:
try:
description = str(
neurovault_row["description"].tolist()[0]).encode("utf-8")
except:
description = ""
if description != "nan":
meta_data["description"] = description
else:
meta_data["description"] = ""
else:
meta_data["description"] = ""
if len(meta_data["description"]) > 600:
meta_data[
"description"] = "%s..." % meta_data["description"][0:600]
output_file = "%s/ri_%s.json" % (base, meta_data["image_id"])
filey = open(output_file, 'wb')
filey.write(
json.dumps(meta_data,
sort_keys=True,
indent=4,
separators=(',', ': ')))
filey.close()
### Concepts
for node in unique_nodes:
# This is a concept node
if not re.search("node_", node):
if node != "1":
relationship_table_row = relationship_table[
relationship_table.id == node]
concept = get_concept(id=node).json
meta_single = {}
children_nodes = [
relationship_table.name.tolist()[x]
for x in range(relationship_table.shape[0])
if relationship_table.parent.tolist()[x] == node
]
while len(
[x for x in children_nodes if not isinstance(x, int)]) > 0:
new_parent_nodes = [
x for x in children_nodes if not isinstance(x, int)
]
children_nodes = [
x for x in children_nodes if x not in new_parent_nodes
]
for new_parent in new_parent_nodes:
node_name = relationship_table.id[
relationship_table.name == new_parent].tolist()[0]
children_nodes = children_nodes + [
relationship_table.name.tolist()[x]
for x in range(relationship_table.shape[0]) if
relationship_table.parent.tolist()[x] == node_name
]
# Now only keep children that are images
meta_single["images"] = images["thumbnail"][
images.image_id.isin(children_nodes)].tolist()
meta_single["image_list"] = children_nodes
# Cognitive Atlas meta data
meta_single[
"url"] = "http://www.cognitiveatlas.org/term/id/%s" % node
meta_single["type"] = "concept"
meta_single[
"thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
meta_single["concept"] = [
relationship_table.name[relationship_table.id ==
node].tolist()[0]
]
meta_single["task"] = ""
meta_single["contrast"] = []
meta_single[
"download"] = "http://www.cognitiveatlas.org/rdf/id/%s" % node
if concept[0]["definition_text"]:
meta_single["description"] = concept[0][
"definition_text"].encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single[
"description"][0:600]
output_file = "%s/ri_%s.json" % (base, node)
filey = open(output_file, 'wb')
filey.write(
json.dumps(meta_single,
sort_keys=True,
indent=4,
separators=(',', ': ')))
filey.close()
if concept[0]["definition_text"]:
meta_single["description"] = concept[0]["definition_text"].encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single["description"][0:600]
meta_data[node] = meta_single
## STEP 2: VISUALIZATION WITH PYBRAINCOMPARE
from pybraincompare.ontology.tree import named_ontology_tree_from_tsv, make_ontology_tree_d3
# First let's look at the tree structure
# output_json = "%s/task_contrast_tree.json" % outfolder
tree = named_ontology_tree_from_tsv(relationship_table,output_json=None,meta_data=meta_data)
html_snippet = make_ontology_tree_d3(tree)
web_folder = "%s/tree" %web
make_analysis_web_folder(html_snippet,web_folder)
# To get a dump of just the tree (for use in more advanced custom web interface)
filey = open('%s/tree/reverseinference.json' %web,'wb')
filey.write(json.dumps(tree, sort_keys=True,indent=4, separators=(',', ': ')))
filey.close()
## STEP 3: Export individual scores
### Images
single_scores_folder = "%s/data/individual_scores" %base # any kind of tsv/result file
single_scores = glob("%s/*.pkl" %single_scores_folder)
scores_export_folder = "%s/indscores" %web
if not os.path.exists(scores_export_folder):
def main():
base = "data/"
# Make a folder for mean images
if not os.path.exists("mr"):
os.mkdir("mr")
# Get Neurovault Images with defined cognitive atlas contrast
collections = get_collections()
# Filter images to those that have a DOI
collections = collections[collections.DOI.isnull()==False]
# Get image meta data for collections (N=1023)
images = get_images(collection_pks=collections.collection_id.tolist())
# Filter images to those with contrasts defined (N=98)
images = images[images.cognitive_contrast_cogatlas_id.isnull()==False]
# Get rid of any not in MNI
images = images[images.not_mni == False]
# Get rid of thresholded images
images = images[images.is_thresholded == False]
### Step 1: Load meta data sources
unique_contrasts = images.cognitive_contrast_cogatlas_id.unique().tolist()
# Images that do not match the correct identifier will not be used (eg, "Other")
expression = re.compile("cnt_*")
unique_contrasts = [u for u in unique_contrasts if expression.match(u)]
# Make sure exists in cognitive atlas
existing_contrasts = []
for u in unique_contrasts:
try:
tmp = get_concept(contrast_id=u,silent=True)
existing_contrasts.append(u)
except:
print "%s is defined in NeuroVault, does not exist in Cognitive Atlas" %u
image_lookup = dict()
for u in existing_contrasts:
image_lookup[u] = images.image_id[images.cognitive_contrast_cogatlas_id==u].tolist()
# Create a data structure of tasks and contrasts for our analysis
relationship_table = concept_node_triples(image_dict=image_lookup,save_to_file=False)
unique_nodes = relationship_table.id.unique().tolist()
# We will store a data frame of meta data
# Lookup for meta_data is the id of the node!
meta_data = {}
for node in unique_nodes:
meta_single = {}
# This is an image node
if re.search("node_",node):
print "Found image node!"
relationship_table_row = relationship_table[relationship_table.id==node]
image_id = relationship_table_row.name.tolist()[0]
meta_single["category"] = ""
meta_single["type"] = "nii"
# NeuroVault metadata
concepts = relationship_table.parent[relationship_table.name == image_id]
concepts = [relationship_table.name[relationship_table.id==c].tolist()[0] for c in concepts]
neurovault_row = images[images.image_id == int(image_id)]
collection_row = collections[collections.collection_id == neurovault_row.collection_id.tolist()[0]]
collection_meta = {"DOI":collection_row["DOI"].tolist()[0],
"authors":collection_row["authors"].tolist()[0],
"journal":collection_row["journal_name"].tolist()[0],
"url":collection_row["url"].tolist()[0],
"subjects":collection_row["number_of_subjects"].tolist()[0],
"smoothing_fwhm":str(collection_row["smoothing_fwhm"].tolist()[0]).encode("utf-8")}
meta_single["url"] = neurovault_row["url"].tolist()[0]
meta_single["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
meta_single["images"] = neurovault_row["thumbnail"].tolist()
meta_single["task"] = neurovault_row["cognitive_paradigm_cogatlas"].tolist()[0]
meta_single["contrast"] = neurovault_row["cognitive_contrast_cogatlas"].tolist()[0]
meta_single["download"] = neurovault_row["file"].tolist()[0]
meta_single["concept"] = concepts
if neurovault_row["description"].tolist()[0]:
meta_single["description"] = str(neurovault_row["description"].tolist()[0]).encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single["description"][0:600]
else: # A concept node
if node != "1":
relationship_table_row = relationship_table[relationship_table.id==node]
concept = get_concept(id=node,silent=True).json
children_nodes = [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node]
while len([x for x in children_nodes if not isinstance(x,int)]) > 0:
new_parent_nodes = [x for x in children_nodes if not isinstance(x,int)]
children_nodes = [x for x in children_nodes if x not in new_parent_nodes]
for new_parent in new_parent_nodes:
node_name = relationship_table.id[relationship_table.name==new_parent].tolist()[0]
children_nodes = children_nodes + [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node_name]
# Now only keep children that are images
meta_single["images"] = images["thumbnail"][images.image_id.isin(children_nodes)].tolist()
# Cognitive Atlas meta data
meta_single["url"] = "http://www.cognitiveatlas.org/term/id/%s" %node
meta_single["type"] = "concept"
meta_single["thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
meta_single["concept"] = [relationship_table.name[relationship_table.id==node].tolist()[0]]
meta_single["task"] = ""
meta_single["contrast"] = []
meta_single["download"] = "http://www.cognitiveatlas.org/rdf/id/%s" %node
if concept[0]["definition_text"]:
meta_single["description"] = concept[0]["definition_text"].encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single["description"][0:600]
meta_data[node] = meta_single
## STEP 2: VISUALIZATION WITH PYBRAINCOMPARE
from pybraincompare.ontology.tree import named_ontology_tree_from_tsv, make_ontology_tree_d3
# First let's look at the tree structure
# output_json = "%s/task_contrast_tree.json" % outfolder
tree = named_ontology_tree_from_tsv(relationship_table,output_json=None,meta_data=meta_data)
html_snippet = make_ontology_tree_d3(tree)
web_folder = base
make_analysis_web_folder(html_snippet,web_folder)
# To get a dump of just the tree (for use in more advanced custom web interface)
filey = open('%s/reverseinference.json' %base,'wb')
filey.write(json.dumps(tree, sort_keys=True,indent=4, separators=(',', ': ')))
filey.close()
## STEP 3: Export individual nodes
### Images
unique_images = images.image_id.unique().tolist()
# Images
for s in range(0,len(unique_images)):
image_id = unique_images[s]
meta_data = {}
meta_data["image_id"] = image_id
print "Parsing data for images %s of %s" %(s,len(unique_images))
concepts = relationship_table.parent[relationship_table.name == str(image_id)].tolist()
concepts = [relationship_table.name[relationship_table.id==c].tolist()[0] for c in concepts]
concepts_ids = [relationship_table.id[relationship_table.id==c].tolist()[0] for c in concepts]
neurovault_row = images[images.image_id == int(image_id)]
collection_row = collections[collections.collection_id == neurovault_row.collection_id.tolist()[0]]
collection_meta = {"DOI":collection_row["DOI"].tolist()[0],
"authors":collection_row["authors"].tolist()[0],
"journal":collection_row["journal_name"].tolist()[0],
"url":collection_row["url"].tolist()[0],
"subjects":collection_row["number_of_subjects"].tolist()[0],
"smoothing_fwhm":str(collection_row["smoothing_fwhm"].tolist()[0]).encode("utf-8"),
"title":collection_row["name"].tolist()[0]}
meta_data["collection"] = collection_meta
meta_data["url"] = neurovault_row["url"].tolist()[0]
meta_data["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
meta_data["images"] = neurovault_row["thumbnail"].tolist()
meta_data["task"] = neurovault_row["cognitive_paradigm_cogatlas"].tolist()[0]
meta_data["contrast"] = neurovault_row["cognitive_contrast_cogatlas"].tolist()[0]
meta_data["download"] = neurovault_row["file"].tolist()[0]
meta_data["concept"] = concepts
meta_data["concept_id"] = concepts_ids
if neurovault_row["description"].tolist()[0]:
try:
description = str(neurovault_row["description"].tolist()[0]).encode("utf-8")
except:
description = ""
if description != "nan":
meta_data["description"] = description
else:
meta_data["description"] = ""
else:
meta_data["description"] = ""
if len(meta_data["description"]) > 600:
meta_data["description"] = "%s..." % meta_data["description"][0:600]
output_file = "%s/ri_%s.json" %(base,meta_data["image_id"])
filey = open(output_file,'wb')
filey.write(json.dumps(meta_data, sort_keys=True,indent=4, separators=(',', ': ')))
filey.close()
### Concepts
for node in unique_nodes:
# This is a concept node
if not re.search("node_",node):
if node != "1":
relationship_table_row = relationship_table[relationship_table.id==node]
concept = get_concept(id=node).json
meta_single = {}
children_nodes = [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node]
while len([x for x in children_nodes if not isinstance(x,int)]) > 0:
new_parent_nodes = [x for x in children_nodes if not isinstance(x,int)]
children_nodes = [x for x in children_nodes if x not in new_parent_nodes]
for new_parent in new_parent_nodes:
node_name = relationship_table.id[relationship_table.name==new_parent].tolist()[0]
children_nodes = children_nodes + [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node_name]
# Now only keep children that are images
meta_single["images"] = images["thumbnail"][images.image_id.isin(children_nodes)].tolist()
meta_single["image_list"] = children_nodes
# Cognitive Atlas meta data
meta_single["url"] = "http://www.cognitiveatlas.org/term/id/%s" %node
meta_single["type"] = "concept"
meta_single["thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
meta_single["concept"] = [relationship_table.name[relationship_table.id==node].tolist()[0]]
meta_single["task"] = ""
meta_single["contrast"] = []
meta_single["download"] = "http://www.cognitiveatlas.org/rdf/id/%s" %node
if concept[0]["definition_text"]:
meta_single["description"] = concept[0]["definition_text"].encode("utf-8")
else:
meta_single["description"] = ""
if len(meta_single["description"]) > 600:
meta_single["description"] = "%s..." % meta_single["description"][0:600]
output_file = "%s/ri_%s.json" %(base,node)
filey = open(output_file,'wb')
filey.write(json.dumps(meta_single, sort_keys=True,indent=4, separators=(',', ': ')))
filey.close()