def setUp(self): self.nc = neo4j_connect( 'http://localhost:7474', 'neo4j', 'neo4j') self.kpw = KB_pattern_writer( 'http://localhost:7474', 'neo4j', 'neo4j') statements = [] for k,v in self.kpw.relation_lookup.items(): short_form = re.split('[/#]', v)[-1] statements.append("MERGE (p:Property { iri : '%s', label: '%s', short_form : '%s' }) " % (v, k, short_form)) for k,v in self.kpw.class_lookup.items(): short_form = re.split('[/#]', v)[-1] statements.append("MERGE (p:Class { iri : '%s', label: '%s', short_form : '%s' }) " % (v, k, short_form)) self.nc.commit_list(statements) statements = [] statements.append("MERGE (p:Class { short_form: 'lobulobus', label: 'lobulobus' })") statements.append("MERGE (p:Individual:Template { short_form: 'template_of_dave', label: 'template_of_dave' })") statements.append("MERGE (s:Site:Individual { short_form : 'fu' }) ") statements.append("MERGE (ds:DataSet:Individual { short_form : 'dosumis2020' }) ") self.nc.commit_list(statements)
def get_lookup(limit_by_prefix=None): if limit_by_prefix: regex_string = ':.+|'.join(limit_by_prefix) + ':.+' where = " AND a.obo_id =~ '%s' " % regex_string else: where = '' nc = neo4j_connect("https://pdb.virtualflybrain.org", "neo4j", "neo4j") lookup_query = "MATCH (a:VFB:Class) WHERE exists (a.obo_id)" + where + " RETURN a.obo_id as id, a.label as name" q = nc.commit_list([lookup_query]) r = results_2_dict_list(q) lookup = {x['name']: x['id'] for x in r} #print(lookup['neuron']) property_query = "MATCH (p:Property) WHERE exists(p.obo_id) RETURN p.obo_id as id, p.label as name" q = nc.commit_list([property_query]) r = results_2_dict_list(q) lookup.update({x['name']: x['id'] for x in r}) #print(lookup['neuron']) return lookup
def gen_simple_report(terms): nc = neo4j_connect("https://pdb.virtualflybrain.org", "neo4j", "neo4j") query = """MATCH (n:Class) WHERE n.iri in %s WITH n OPTIONAL MATCH (n)-[r]->(p:pub) WHERE r.typ = 'syn' WITH n, COLLECT({ synonym: r.synonym, PMID: 'PMID:' + p.PMID, miniref: p.label}) AS syns OPTIONAL MATCH (n)-[r]-(p:pub) WHERE r.typ = 'def' with n, syns, collect({ PMID: 'PMID:' + p.PMID, miniref: p.label}) as pubs OPTIONAL MATCH (n)-[:SUBCLASSOF]->(super:Class) RETURN n.short_form as short_form, n.label as label, n.description as description, syns, pubs, super.label, super.short_form """ % str(terms) #print(query) q = nc.commit_list([query]) return results_2_dict_list(q)
def gen_report(server, query, report_name, column_order=None): """Generates a pandas dataframe with the results of a cypher query against the specified server. Args: server: server connection as [endpoint, usr, pwd] query: cypher query report_name: df.name column_order: optionally specify column order in df.""" nc = neo4j_connect(*server) print(query) r = nc.commit_list([query]) #print(r) dc = results_2_dict_list(r) report = pd.DataFrame.from_records(dc) report.replace(np.nan, '', regex=True, inplace=True) report.name = report_name if column_order: out = report[column_order] out.name = report_name return out else: return report
from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect, results_2_dict_list import sys import json import re nc = neo4j_connect(base_uri=sys.argv[1], usr=sys.argv[2], pwd=sys.argv[3]) """ Converts references on definitions and synonyms, stored as entity attributes in OLS Neo4j, into edges linked to pub nodes. In the case of synonyms, edges store synonym names, scopes and types. Background: OLS Neo4J includes references attached to definitions and synonyms, but these are packed into JSON strings on attributes. Almost every reference has an FBrf, but a few only have PMIDS or DOIs. Merge strategy uses FBrf first, then PMID, then DOI.""" supported_xrefs = { 'FlyBase': 'FlyBase:FBrf\d{7}', 'PMID': 'PMID:\d+', 'DOI': 'DOI:.+', 'http': 'http:.+' } # obo_definition_citation:{"definition":"Any sense organ (FBbt:00005155) that is part of some adult (FBbt:00003004).", # "oboXrefs":[{"database":"FlyBase","id":"FBrf0031004","description":null,"url":null}, # {"database":"FlyBase","id":"FBrf0007734","description":null,"url":null}, # {"database":"FBC","id":"auto_generated_definition","description":null,"url":null}]}
import pandas as pd from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect, results_2_dict_list import get_catmaid_papers nc = neo4j_connect('http://kb.virtualflybrain.org', 'neo4j', 'neo4j') # variables for generating reports # dict of sources and project IDs for larval datasets larval_sources = { 'l1em': 1, 'abd1.5': 1, 'iav-robo': 1, 'iav-tnt': 2, 'l3vnc': 2 } larval_reports = [[ get_catmaid_papers.gen_cat_paper_report( "https://" + s + ".catmaid.virtualflybrain.org", larval_sources[s], "papers"), get_catmaid_papers.gen_cat_skid_report_officialnames( "https://" + s + ".catmaid.virtualflybrain.org", larval_sources[s], "papers", ["neuron name", "MB nomenclature"]), s.upper() ] for s in larval_sources.keys()] FAFB = [ get_catmaid_papers.gen_cat_paper_report( "https://fafb.catmaid.virtualflybrain.org", 1, "Published"), get_catmaid_papers.gen_cat_skid_report_officialnames( "https://fafb.catmaid.virtualflybrain.org", 1, "Published",
parser = argparse.ArgumentParser() parser.add_argument('--test', help='Run in test mode. ' \ 'runs with limits on cypher queries and additions.', action = "store_true") parser.add_argument("endpoint", help="Endpoint for connection to neo4J prod") parser.add_argument("usr", help="username") parser.add_argument("pwd", help="password") args = parser.parse_args() # Current version makes all edges. Might want to limit the types of edges made to those needed for graphing purposes. # TODO: add in check of uniqueness constraint # Use REST calls to /db/data/schema/ nc = neo4j_connect(base_uri=args.endpoint, usr=args.usr, pwd=args.pwd) # def make_name_edges(typ, s='', o='', test_mode = False): # if test_mode: # test = " limit 10" # else: # test = "" # """ typ = edge label. o, s = subject and object labels. These hould be pre prepended with ':'""" # statements = ["MATCH (n%s)-[r:%s]->(m%s) RETURN n.short_form, r.label, m.short_form %s" % (s, typ, o, test)] # r = nc.commit_list(statements) # triples = [x['row'] for x in r[0]['data']] # statements = [] # # Iterate over, making named edges for labels (sub space for _) # print("Processing %d triples" % len(triples)) # for t in triples: # subj = t[0]
import sys from uk.ac.ebi.vfb.neo4j.flybase2neo.fb_tools import dict_cursor, get_fb_conn from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect import re """Populate pub data. Should be run as a final step, once all content added.""" ## TODO: Add pub relationships (P3) base_uri = sys.argv[1] usr = sys.argv[2] pwd = sys.argv[3] nc = neo4j_connect(base_uri, usr, pwd) # Pull all pub FBrfs from graph statements = ['MATCH (p:pub) RETURN DISTINCT p.FlyBase'] pub_list_results = nc.commit_list(statements) pub_list = [str(x['row'][0]) for x in pub_list_results[0]['data'] ] # Parsing returned Json for results. c = get_fb_conn() cursor = c.cursor() def gen_micro_ref_from_miniref(miniref): # Use regex to truncate after year, remove brackets. return def gen_micro_ref_from_db():
# Merge strategy 1: # Merge requires all labels on nodes retrieved from KB to match those those on nodes with same key in prod # Scenario 1: Node loaded from OWL, has extra labels in KB # KB Load will fail. # Merge strategy 2 # If loading edge to prod loads nodes through merge => duplicate nodes. # Solution is to align node labels from KB -> Prod prior to edge loading. # But how to do this safely? Can't just match on iri! # Two options: # - Implement a label mover that works with match statements. Run this with equivalent queries before # running edge loader # - Break up match statement on edge mover into s, r, o chunks. Use the s and o chunks to specify edges to move. # Limitation of this approach is that only simple match statements or possible # - with just a triple and no 'where' clause. kb = neo4j_connect(sys.argv[1], sys.argv[2], sys.argv[3]) prod = neo4j_connect(sys.argv[4], sys.argv[5], sys.argv[6]) ni = node_importer(sys.argv[4], sys.argv[5], sys.argv[6]) ni.add_default_constraint_set(['DataSet', 'Site', 'License', 'Individual']) ncm = neo4jContentMover(kb, prod) # Labels may not match on nodes already loaded. The following finds all Individuals in KB that are in Prod, # checks for differences in labels, and adds them to the equivalent node in Prod if they match. ncm.move_node_labels(match="MATCH (n:Individual) ", node_key='iri') ## move channels and directly associated edges ## Move all instance of channel:
from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect, results_2_dict_list import sys import warnings import json # KB tests # Query for dataSets (would be helpful if these were tagged as image) nc = neo4j_connect(sys.argv[1], sys.argv[2], sys.argv[3]) def query(query): q = nc.commit_list([query]) if not q: return False dc = results_2_dict_list(q) if not dc: return False else: return dc def query_ind_count(query): q = nc.commit_list([query]) if not q: return False dc = results_2_dict_list(q) if not dc: return False if not ('ind_count' in dc[0].keys()): warnings.warn("Query has no ind_count") return False
def setUp(self): To = neo4j_connect('http://localhost:7474', 'neo4j', 'neo4j') From = neo4j_connect('http://kb.virtualflybrain.org', 'neo4j', 'neo4j') self.ncm = neo4jContentMover(From=From, To=To)
def setUp(self): self.nc = neo4j_connect('http://localhost:7474', 'neo4j', 'neo4j')
def _init(self, endpoint, usr, pwd): self.conn = get_fb_conn() self.nc = neo4j_connect(endpoint, usr, pwd)