예제 #1
0
def main():

    base = "data/"

    # Make a folder for mean images
    if not os.path.exists("mr"):
        os.mkdir("mr")

    # Get Neurovault Images with defined cognitive atlas contrast
    collections = get_collections()

    # Filter images to those that have a DOI
    collections = collections[collections.DOI.isnull() == False]

    # Get image meta data for collections (N=1023)
    images = get_images(collection_pks=collections.collection_id.tolist())

    # Filter images to those with contrasts defined (N=98)
    images = images[images.cognitive_contrast_cogatlas_id.isnull() == False]

    # Get rid of any not in MNI
    images = images[images.not_mni == False]

    # Get rid of thresholded images
    images = images[images.is_thresholded == False]

    ### Step 1: Load meta data sources
    unique_contrasts = images.cognitive_contrast_cogatlas_id.unique().tolist()

    # Images that do not match the correct identifier will not be used (eg, "Other")
    expression = re.compile("cnt_*")
    unique_contrasts = [u for u in unique_contrasts if expression.match(u)]

    # Make sure exists in cognitive atlas
    existing_contrasts = []
    for u in unique_contrasts:
        try:
            tmp = get_concept(contrast_id=u, silent=True)
            existing_contrasts.append(u)
        except:
            print "%s is defined in NeuroVault, does not exist in Cognitive Atlas" % u

    image_lookup = dict()
    for u in existing_contrasts:
        image_lookup[u] = images.image_id[images.cognitive_contrast_cogatlas_id
                                          == u].tolist()

    # Create a data structure of tasks and contrasts for our analysis
    relationship_table = concept_node_triples(image_dict=image_lookup,
                                              save_to_file=False)

    unique_nodes = relationship_table.id.unique().tolist()

    # We will store a data frame of meta data
    # Lookup for meta_data is the id of the node!
    meta_data = {}

    for node in unique_nodes:
        meta_single = {}
        # This is an image node
        if re.search("node_", node):
            print "Found image node!"
            relationship_table_row = relationship_table[relationship_table.id
                                                        == node]
            image_id = relationship_table_row.name.tolist()[0]
            meta_single["category"] = ""
            meta_single["type"] = "nii"
            # NeuroVault metadata
            concepts = relationship_table.parent[relationship_table.name ==
                                                 image_id]
            concepts = [
                relationship_table.name[relationship_table.id == c].tolist()[0]
                for c in concepts
            ]
            neurovault_row = images[images.image_id == int(image_id)]
            collection_row = collections[collections.collection_id ==
                                         neurovault_row.collection_id.tolist()
                                         [0]]
            collection_meta = {
                "DOI":
                collection_row["DOI"].tolist()[0],
                "authors":
                collection_row["authors"].tolist()[0],
                "journal":
                collection_row["journal_name"].tolist()[0],
                "url":
                collection_row["url"].tolist()[0],
                "subjects":
                collection_row["number_of_subjects"].tolist()[0],
                "smoothing_fwhm":
                str(collection_row["smoothing_fwhm"].tolist()[0]).encode(
                    "utf-8")
            }
            meta_single["url"] = neurovault_row["url"].tolist()[0]
            meta_single["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
            meta_single["images"] = neurovault_row["thumbnail"].tolist()
            meta_single["task"] = neurovault_row[
                "cognitive_paradigm_cogatlas"].tolist()[0]
            meta_single["contrast"] = neurovault_row[
                "cognitive_contrast_cogatlas"].tolist()[0]
            meta_single["download"] = neurovault_row["file"].tolist()[0]
            meta_single["concept"] = concepts
            if neurovault_row["description"].tolist()[0]:
                meta_single["description"] = str(
                    neurovault_row["description"].tolist()[0]).encode("utf-8")
            else:
                meta_single["description"] = ""
            if len(meta_single["description"]) > 600:
                meta_single["description"] = "%s..." % meta_single[
                    "description"][0:600]
        else:  # A concept node
            if node != "1":
                relationship_table_row = relationship_table[
                    relationship_table.id == node]
                concept = get_concept(id=node, silent=True).json
                children_nodes = [
                    relationship_table.name.tolist()[x]
                    for x in range(relationship_table.shape[0])
                    if relationship_table.parent.tolist()[x] == node
                ]
                while len(
                    [x for x in children_nodes if not isinstance(x, int)]) > 0:
                    new_parent_nodes = [
                        x for x in children_nodes if not isinstance(x, int)
                    ]
                    children_nodes = [
                        x for x in children_nodes if x not in new_parent_nodes
                    ]
                    for new_parent in new_parent_nodes:
                        node_name = relationship_table.id[
                            relationship_table.name == new_parent].tolist()[0]
                        children_nodes = children_nodes + [
                            relationship_table.name.tolist()[x]
                            for x in range(relationship_table.shape[0]) if
                            relationship_table.parent.tolist()[x] == node_name
                        ]
                # Now only keep children that are images
                meta_single["images"] = images["thumbnail"][
                    images.image_id.isin(children_nodes)].tolist()
                # Cognitive Atlas meta data
                meta_single[
                    "url"] = "http://www.cognitiveatlas.org/term/id/%s" % node
                meta_single["type"] = "concept"
                meta_single[
                    "thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
                meta_single["concept"] = [
                    relationship_table.name[relationship_table.id ==
                                            node].tolist()[0]
                ]
                meta_single["task"] = ""
                meta_single["contrast"] = []
                meta_single[
                    "download"] = "http://www.cognitiveatlas.org/rdf/id/%s" % node
                if concept[0]["definition_text"]:
                    meta_single["description"] = concept[0][
                        "definition_text"].encode("utf-8")
                else:
                    meta_single["description"] = ""
                if len(meta_single["description"]) > 600:
                    meta_single["description"] = "%s..." % meta_single[
                        "description"][0:600]
        meta_data[node] = meta_single

    ## STEP 2: VISUALIZATION WITH PYBRAINCOMPARE
    from pybraincompare.ontology.tree import named_ontology_tree_from_tsv, make_ontology_tree_d3

    # First let's look at the tree structure
    # output_json = "%s/task_contrast_tree.json" % outfolder
    tree = named_ontology_tree_from_tsv(relationship_table,
                                        output_json=None,
                                        meta_data=meta_data)
    html_snippet = make_ontology_tree_d3(tree)
    web_folder = base
    make_analysis_web_folder(html_snippet, web_folder)

    # To get a dump of just the tree (for use in more advanced custom web interface)
    filey = open('%s/reverseinference.json' % base, 'wb')
    filey.write(
        json.dumps(tree, sort_keys=True, indent=4, separators=(',', ': ')))
    filey.close()

    ## STEP 3: Export individual nodes

    ### Images
    unique_images = images.image_id.unique().tolist()

    # Images
    for s in range(0, len(unique_images)):
        image_id = unique_images[s]
        meta_data = {}
        meta_data["image_id"] = image_id
        print "Parsing data for images %s of %s" % (s, len(unique_images))
        concepts = relationship_table.parent[relationship_table.name == str(
            image_id)].tolist()
        concepts = [
            relationship_table.name[relationship_table.id == c].tolist()[0]
            for c in concepts
        ]
        concepts_ids = [
            relationship_table.id[relationship_table.id == c].tolist()[0]
            for c in concepts
        ]
        neurovault_row = images[images.image_id == int(image_id)]
        collection_row = collections[collections.collection_id ==
                                     neurovault_row.collection_id.tolist()[0]]
        collection_meta = {
            "DOI":
            collection_row["DOI"].tolist()[0],
            "authors":
            collection_row["authors"].tolist()[0],
            "journal":
            collection_row["journal_name"].tolist()[0],
            "url":
            collection_row["url"].tolist()[0],
            "subjects":
            collection_row["number_of_subjects"].tolist()[0],
            "smoothing_fwhm":
            str(collection_row["smoothing_fwhm"].tolist()[0]).encode("utf-8"),
            "title":
            collection_row["name"].tolist()[0]
        }
        meta_data["collection"] = collection_meta
        meta_data["url"] = neurovault_row["url"].tolist()[0]
        meta_data["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
        meta_data["images"] = neurovault_row["thumbnail"].tolist()
        meta_data["task"] = neurovault_row[
            "cognitive_paradigm_cogatlas"].tolist()[0]
        meta_data["contrast"] = neurovault_row[
            "cognitive_contrast_cogatlas"].tolist()[0]
        meta_data["download"] = neurovault_row["file"].tolist()[0]
        meta_data["concept"] = concepts
        meta_data["concept_id"] = concepts_ids
        if neurovault_row["description"].tolist()[0]:
            try:
                description = str(
                    neurovault_row["description"].tolist()[0]).encode("utf-8")
            except:
                description = ""
            if description != "nan":
                meta_data["description"] = description
            else:
                meta_data["description"] = ""
        else:
            meta_data["description"] = ""
        if len(meta_data["description"]) > 600:
            meta_data[
                "description"] = "%s..." % meta_data["description"][0:600]
        output_file = "%s/ri_%s.json" % (base, meta_data["image_id"])
        filey = open(output_file, 'wb')
        filey.write(
            json.dumps(meta_data,
                       sort_keys=True,
                       indent=4,
                       separators=(',', ': ')))
        filey.close()

    ### Concepts
    for node in unique_nodes:
        # This is a concept node
        if not re.search("node_", node):
            if node != "1":
                relationship_table_row = relationship_table[
                    relationship_table.id == node]
                concept = get_concept(id=node).json
                meta_single = {}
                children_nodes = [
                    relationship_table.name.tolist()[x]
                    for x in range(relationship_table.shape[0])
                    if relationship_table.parent.tolist()[x] == node
                ]
                while len(
                    [x for x in children_nodes if not isinstance(x, int)]) > 0:
                    new_parent_nodes = [
                        x for x in children_nodes if not isinstance(x, int)
                    ]
                    children_nodes = [
                        x for x in children_nodes if x not in new_parent_nodes
                    ]
                    for new_parent in new_parent_nodes:
                        node_name = relationship_table.id[
                            relationship_table.name == new_parent].tolist()[0]
                        children_nodes = children_nodes + [
                            relationship_table.name.tolist()[x]
                            for x in range(relationship_table.shape[0]) if
                            relationship_table.parent.tolist()[x] == node_name
                        ]
                # Now only keep children that are images
                meta_single["images"] = images["thumbnail"][
                    images.image_id.isin(children_nodes)].tolist()
                meta_single["image_list"] = children_nodes
                # Cognitive Atlas meta data
                meta_single[
                    "url"] = "http://www.cognitiveatlas.org/term/id/%s" % node
                meta_single["type"] = "concept"
                meta_single[
                    "thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
                meta_single["concept"] = [
                    relationship_table.name[relationship_table.id ==
                                            node].tolist()[0]
                ]
                meta_single["task"] = ""
                meta_single["contrast"] = []
                meta_single[
                    "download"] = "http://www.cognitiveatlas.org/rdf/id/%s" % node
                if concept[0]["definition_text"]:
                    meta_single["description"] = concept[0][
                        "definition_text"].encode("utf-8")
                else:
                    meta_single["description"] = ""
                if len(meta_single["description"]) > 600:
                    meta_single["description"] = "%s..." % meta_single[
                        "description"][0:600]
                output_file = "%s/ri_%s.json" % (base, node)
                filey = open(output_file, 'wb')
                filey.write(
                    json.dumps(meta_single,
                               sort_keys=True,
                               indent=4,
                               separators=(',', ': ')))
                filey.close()
            if concept[0]["definition_text"]:
                meta_single["description"] = concept[0]["definition_text"].encode("utf-8")
            else:
                meta_single["description"] = ""
            if len(meta_single["description"]) > 600:
                meta_single["description"] = "%s..." % meta_single["description"][0:600]
    meta_data[node] = meta_single


## STEP 2: VISUALIZATION WITH PYBRAINCOMPARE
from pybraincompare.ontology.tree import named_ontology_tree_from_tsv, make_ontology_tree_d3

# First let's look at the tree structure
# output_json = "%s/task_contrast_tree.json" % outfolder
tree = named_ontology_tree_from_tsv(relationship_table,output_json=None,meta_data=meta_data)
html_snippet = make_ontology_tree_d3(tree)
web_folder = "%s/tree" %web
make_analysis_web_folder(html_snippet,web_folder)

# To get a dump of just the tree (for use in more advanced custom web interface)
filey = open('%s/tree/reverseinference.json' %web,'wb')
filey.write(json.dumps(tree, sort_keys=True,indent=4, separators=(',', ': ')))
filey.close()

## STEP 3: Export individual scores

### Images
single_scores_folder = "%s/data/individual_scores" %base  # any kind of tsv/result file
single_scores = glob("%s/*.pkl" %single_scores_folder)
scores_export_folder = "%s/indscores" %web
if not os.path.exists(scores_export_folder):
예제 #3
0
def main():

    base = "data/"

    # Make a folder for mean images
    if not os.path.exists("mr"):
        os.mkdir("mr")

    # Get Neurovault Images with defined cognitive atlas contrast    
    collections = get_collections()

    # Filter images to those that have a DOI
    collections = collections[collections.DOI.isnull()==False]
    
    # Get image meta data for collections (N=1023)
    images = get_images(collection_pks=collections.collection_id.tolist())

    # Filter images to those with contrasts defined (N=98)
    images = images[images.cognitive_contrast_cogatlas_id.isnull()==False]

    # Get rid of any not in MNI
    images = images[images.not_mni == False]

    # Get rid of thresholded images
    images = images[images.is_thresholded == False]

    ### Step 1: Load meta data sources 
    unique_contrasts = images.cognitive_contrast_cogatlas_id.unique().tolist()

    # Images that do not match the correct identifier will not be used (eg, "Other")
    expression = re.compile("cnt_*")
    unique_contrasts = [u for u in unique_contrasts if expression.match(u)]

    # Make sure exists in cognitive atlas
    existing_contrasts = []
    for u in unique_contrasts:
        try:
           tmp = get_concept(contrast_id=u,silent=True)
           existing_contrasts.append(u)
        except:
            print "%s is defined in NeuroVault, does not exist in Cognitive Atlas" %u

    image_lookup = dict()
    for u in existing_contrasts:
        image_lookup[u] = images.image_id[images.cognitive_contrast_cogatlas_id==u].tolist()

    # Create a data structure of tasks and contrasts for our analysis
    relationship_table = concept_node_triples(image_dict=image_lookup,save_to_file=False)

    unique_nodes = relationship_table.id.unique().tolist()

    # We will store a data frame of meta data
    # Lookup for meta_data is the id of the node!
    meta_data = {}

    for node in unique_nodes:
        meta_single = {}
        # This is an image node
        if re.search("node_",node):
            print "Found image node!"
            relationship_table_row = relationship_table[relationship_table.id==node]
            image_id = relationship_table_row.name.tolist()[0]
            meta_single["category"] = ""
            meta_single["type"] = "nii"
            # NeuroVault metadata
            concepts = relationship_table.parent[relationship_table.name == image_id]
            concepts = [relationship_table.name[relationship_table.id==c].tolist()[0] for c in concepts]
            neurovault_row = images[images.image_id == int(image_id)]
            collection_row = collections[collections.collection_id == neurovault_row.collection_id.tolist()[0]]
            collection_meta = {"DOI":collection_row["DOI"].tolist()[0],
                               "authors":collection_row["authors"].tolist()[0],
                               "journal":collection_row["journal_name"].tolist()[0],
                               "url":collection_row["url"].tolist()[0],
                               "subjects":collection_row["number_of_subjects"].tolist()[0],
                               "smoothing_fwhm":str(collection_row["smoothing_fwhm"].tolist()[0]).encode("utf-8")}
            meta_single["url"] = neurovault_row["url"].tolist()[0]
            meta_single["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
            meta_single["images"] = neurovault_row["thumbnail"].tolist()
            meta_single["task"] = neurovault_row["cognitive_paradigm_cogatlas"].tolist()[0]
            meta_single["contrast"] = neurovault_row["cognitive_contrast_cogatlas"].tolist()[0]
            meta_single["download"] = neurovault_row["file"].tolist()[0]
            meta_single["concept"] = concepts
            if neurovault_row["description"].tolist()[0]:
                meta_single["description"] =  str(neurovault_row["description"].tolist()[0]).encode("utf-8")
            else:
                meta_single["description"] = ""
            if len(meta_single["description"]) > 600:
                meta_single["description"] = "%s..." % meta_single["description"][0:600]
        else: # A concept node
            if node != "1":
                relationship_table_row = relationship_table[relationship_table.id==node]
                concept = get_concept(id=node,silent=True).json
                children_nodes = [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node]
                while len([x for x in children_nodes if not isinstance(x,int)]) > 0:
                    new_parent_nodes = [x for x in children_nodes if not isinstance(x,int)]
                    children_nodes = [x for x in children_nodes if x not in new_parent_nodes]
                    for new_parent in new_parent_nodes:
                        node_name = relationship_table.id[relationship_table.name==new_parent].tolist()[0]
                        children_nodes = children_nodes + [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node_name]
                # Now only keep children that are images
                meta_single["images"] = images["thumbnail"][images.image_id.isin(children_nodes)].tolist()
                # Cognitive Atlas meta data
                meta_single["url"] = "http://www.cognitiveatlas.org/term/id/%s" %node
                meta_single["type"] = "concept"
                meta_single["thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
                meta_single["concept"] = [relationship_table.name[relationship_table.id==node].tolist()[0]]
                meta_single["task"] = ""
                meta_single["contrast"] = []
                meta_single["download"] = "http://www.cognitiveatlas.org/rdf/id/%s" %node
                if concept[0]["definition_text"]:
                    meta_single["description"] = concept[0]["definition_text"].encode("utf-8")
                else:
                    meta_single["description"] = ""
                if len(meta_single["description"]) > 600:
                    meta_single["description"] = "%s..." % meta_single["description"][0:600]
        meta_data[node] = meta_single
    
    
    ## STEP 2: VISUALIZATION WITH PYBRAINCOMPARE
    from pybraincompare.ontology.tree import named_ontology_tree_from_tsv, make_ontology_tree_d3

    # First let's look at the tree structure
    # output_json = "%s/task_contrast_tree.json" % outfolder
    tree = named_ontology_tree_from_tsv(relationship_table,output_json=None,meta_data=meta_data)
    html_snippet = make_ontology_tree_d3(tree)
    web_folder = base
    make_analysis_web_folder(html_snippet,web_folder)

    # To get a dump of just the tree (for use in more advanced custom web interface)
    filey = open('%s/reverseinference.json' %base,'wb')
    filey.write(json.dumps(tree, sort_keys=True,indent=4, separators=(',', ': ')))
    filey.close()

    ## STEP 3: Export individual nodes

    ### Images
    unique_images = images.image_id.unique().tolist()

    # Images
    for s in range(0,len(unique_images)):
        image_id = unique_images[s]
        meta_data = {}
        meta_data["image_id"] = image_id
        print "Parsing data for images %s of %s" %(s,len(unique_images))
        concepts = relationship_table.parent[relationship_table.name == str(image_id)].tolist()
        concepts = [relationship_table.name[relationship_table.id==c].tolist()[0] for c in concepts]
        concepts_ids = [relationship_table.id[relationship_table.id==c].tolist()[0] for c in concepts]
        neurovault_row = images[images.image_id == int(image_id)]            
        collection_row = collections[collections.collection_id == neurovault_row.collection_id.tolist()[0]]
        collection_meta = {"DOI":collection_row["DOI"].tolist()[0],
                           "authors":collection_row["authors"].tolist()[0],
                           "journal":collection_row["journal_name"].tolist()[0],
                           "url":collection_row["url"].tolist()[0],
                           "subjects":collection_row["number_of_subjects"].tolist()[0],
                           "smoothing_fwhm":str(collection_row["smoothing_fwhm"].tolist()[0]).encode("utf-8"),
                           "title":collection_row["name"].tolist()[0]}
        meta_data["collection"] = collection_meta
        meta_data["url"] = neurovault_row["url"].tolist()[0]
        meta_data["thumbnail"] = neurovault_row["thumbnail"].tolist()[0]
        meta_data["images"] = neurovault_row["thumbnail"].tolist()
        meta_data["task"] = neurovault_row["cognitive_paradigm_cogatlas"].tolist()[0]
        meta_data["contrast"] = neurovault_row["cognitive_contrast_cogatlas"].tolist()[0]
        meta_data["download"] = neurovault_row["file"].tolist()[0]
        meta_data["concept"] = concepts
        meta_data["concept_id"] = concepts_ids
        if neurovault_row["description"].tolist()[0]:
            try:
                description = str(neurovault_row["description"].tolist()[0]).encode("utf-8")
            except:
                description = ""
            if description != "nan":
                meta_data["description"] =  description
            else:
                meta_data["description"] = ""
        else:
            meta_data["description"] = ""
        if len(meta_data["description"]) > 600:
            meta_data["description"] = "%s..." % meta_data["description"][0:600]
        output_file = "%s/ri_%s.json" %(base,meta_data["image_id"])
        filey = open(output_file,'wb')
        filey.write(json.dumps(meta_data, sort_keys=True,indent=4, separators=(',', ': ')))
        filey.close()
    

    ### Concepts
    for node in unique_nodes:
        # This is a concept node
        if not re.search("node_",node):
            if node != "1":
                relationship_table_row = relationship_table[relationship_table.id==node]
                concept = get_concept(id=node).json
                meta_single = {}
                children_nodes = [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node]
                while len([x for x in children_nodes if not isinstance(x,int)]) > 0:
                    new_parent_nodes = [x for x in children_nodes if not isinstance(x,int)]
                    children_nodes = [x for x in children_nodes if x not in new_parent_nodes]
                    for new_parent in new_parent_nodes:
                        node_name = relationship_table.id[relationship_table.name==new_parent].tolist()[0]
                        children_nodes = children_nodes + [relationship_table.name.tolist()[x] for x in range(relationship_table.shape[0]) if relationship_table.parent.tolist()[x]==node_name]
                # Now only keep children that are images
                meta_single["images"] = images["thumbnail"][images.image_id.isin(children_nodes)].tolist()
                meta_single["image_list"] = children_nodes
                # Cognitive Atlas meta data
                meta_single["url"] = "http://www.cognitiveatlas.org/term/id/%s" %node
                meta_single["type"] = "concept"
                meta_single["thumbnail"] = "http://www.cognitiveatlas.org/images/logo-front.png"
                meta_single["concept"] = [relationship_table.name[relationship_table.id==node].tolist()[0]]
                meta_single["task"] = ""
                meta_single["contrast"] = []
                meta_single["download"] = "http://www.cognitiveatlas.org/rdf/id/%s" %node
                if concept[0]["definition_text"]:
                    meta_single["description"] = concept[0]["definition_text"].encode("utf-8")
                else:
                    meta_single["description"] = ""
                if len(meta_single["description"]) > 600:
                    meta_single["description"] = "%s..." % meta_single["description"][0:600]
                output_file = "%s/ri_%s.json" %(base,node)
                filey = open(output_file,'wb')
                filey.write(json.dumps(meta_single, sort_keys=True,indent=4, separators=(',', ': ')))
                filey.close()