def setUp(self):
self.nc = neo4j_connect(
'http://localhost:7474', 'neo4j', 'neo4j')
self.kpw = KB_pattern_writer(
'http://localhost:7474', 'neo4j', 'neo4j')
statements = []
for k,v in self.kpw.relation_lookup.items():
short_form = re.split('[/#]', v)[-1]
statements.append("MERGE (p:Property { iri : '%s', label: '%s', short_form : '%s' }) " %
(v, k, short_form))
for k,v in self.kpw.class_lookup.items():
short_form = re.split('[/#]', v)[-1]
statements.append("MERGE (p:Class { iri : '%s', label: '%s', short_form : '%s' }) " %
(v, k, short_form))
self.nc.commit_list(statements)
statements = []
statements.append("MERGE (p:Class { short_form: 'lobulobus', label: 'lobulobus' })")
statements.append("MERGE (p:Individual:Template { short_form: 'template_of_dave', label: 'template_of_dave' })")
statements.append("MERGE (s:Site:Individual { short_form : 'fu' }) ")
statements.append("MERGE (ds:DataSet:Individual { short_form : 'dosumis2020' }) ")
self.nc.commit_list(statements)
def get_lookup(limit_by_prefix=None):
if limit_by_prefix:
regex_string = ':.+|'.join(limit_by_prefix) + ':.+'
where = " AND a.obo_id =~ '%s' " % regex_string
else:
where = ''
nc = neo4j_connect("https://pdb.virtualflybrain.org", "neo4j", "neo4j")
lookup_query = "MATCH (a:VFB:Class) WHERE exists (a.obo_id)" + where + " RETURN a.obo_id as id, a.label as name"
q = nc.commit_list([lookup_query])
r = results_2_dict_list(q)
lookup = {x['name']: x['id'] for x in r}
#print(lookup['neuron'])
property_query = "MATCH (p:Property) WHERE exists(p.obo_id) RETURN p.obo_id as id, p.label as name"
q = nc.commit_list([property_query])
r = results_2_dict_list(q)
lookup.update({x['name']: x['id'] for x in r})
#print(lookup['neuron'])
return lookup
def gen_simple_report(terms):
nc = neo4j_connect("https://pdb.virtualflybrain.org", "neo4j", "neo4j")
query = """MATCH (n:Class) WHERE n.iri in %s WITH n
OPTIONAL MATCH (n)-[r]->(p:pub) WHERE r.typ = 'syn'
WITH n,
COLLECT({ synonym: r.synonym, PMID: 'PMID:' + p.PMID,
miniref: p.label}) AS syns
OPTIONAL MATCH (n)-[r]-(p:pub) WHERE r.typ = 'def'
with n, syns,
collect({ PMID: 'PMID:' + p.PMID, miniref: p.label}) as pubs
OPTIONAL MATCH (n)-[:SUBCLASSOF]->(super:Class)
RETURN n.short_form as short_form, n.label as label,
n.description as description, syns, pubs,
super.label, super.short_form
""" % str(terms)
#print(query)
q = nc.commit_list([query])
return results_2_dict_list(q)
def gen_report(server, query, report_name, column_order=None):
"""Generates a pandas dataframe with
the results of a cypher query against the
specified server.
Args:
server: server connection as [endpoint, usr, pwd]
query: cypher query
report_name: df.name
column_order: optionally specify column order in df."""
nc = neo4j_connect(*server)
print(query)
r = nc.commit_list([query])
#print(r)
dc = results_2_dict_list(r)
report = pd.DataFrame.from_records(dc)
report.replace(np.nan, '', regex=True, inplace=True)
report.name = report_name
if column_order:
out = report[column_order]
out.name = report_name
return out
else:
return report
from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect, results_2_dict_list
import sys
import json
import re
nc = neo4j_connect(base_uri=sys.argv[1], usr=sys.argv[2], pwd=sys.argv[3])
"""
Converts references on definitions and synonyms, stored as entity attributes
in OLS Neo4j, into edges linked to pub nodes. In the case of synonyms,
edges store synonym names, scopes and types.
Background:
OLS Neo4J includes references attached to definitions and synonyms,
but these are packed into JSON strings on attributes.
Almost every reference has an FBrf, but a few only have PMIDS or DOIs.
Merge strategy uses FBrf first, then PMID, then DOI."""
supported_xrefs = {
'FlyBase': 'FlyBase:FBrf\d{7}',
'PMID': 'PMID:\d+',
'DOI': 'DOI:.+',
'http': 'http:.+'
}
# obo_definition_citation:{"definition":"Any sense organ (FBbt:00005155) that is part of some adult (FBbt:00003004).",
# "oboXrefs":[{"database":"FlyBase","id":"FBrf0031004","description":null,"url":null},
# {"database":"FlyBase","id":"FBrf0007734","description":null,"url":null},
# {"database":"FBC","id":"auto_generated_definition","description":null,"url":null}]}
import pandas as pd
from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect, results_2_dict_list
import get_catmaid_papers
nc = neo4j_connect('http://kb.virtualflybrain.org', 'neo4j', 'neo4j')
# variables for generating reports
# dict of sources and project IDs for larval datasets
larval_sources = {
'l1em': 1,
'abd1.5': 1,
'iav-robo': 1,
'iav-tnt': 2,
'l3vnc': 2
}
larval_reports = [[
get_catmaid_papers.gen_cat_paper_report(
"https://" + s + ".catmaid.virtualflybrain.org", larval_sources[s],
"papers"),
get_catmaid_papers.gen_cat_skid_report_officialnames(
"https://" + s + ".catmaid.virtualflybrain.org", larval_sources[s],
"papers", ["neuron name", "MB nomenclature"]),
s.upper()
] for s in larval_sources.keys()]
FAFB = [
get_catmaid_papers.gen_cat_paper_report(
"https://fafb.catmaid.virtualflybrain.org", 1, "Published"),
get_catmaid_papers.gen_cat_skid_report_officialnames(
"https://fafb.catmaid.virtualflybrain.org", 1, "Published",
parser = argparse.ArgumentParser()
parser.add_argument('--test', help='Run in test mode. ' \
'runs with limits on cypher queries and additions.',
action = "store_true")
parser.add_argument("endpoint", help="Endpoint for connection to neo4J prod")
parser.add_argument("usr", help="username")
parser.add_argument("pwd", help="password")
args = parser.parse_args()
# Current version makes all edges. Might want to limit the types of edges made to those needed for graphing purposes.
# TODO: add in check of uniqueness constraint
# Use REST calls to /db/data/schema/
nc = neo4j_connect(base_uri=args.endpoint, usr=args.usr, pwd=args.pwd)
# def make_name_edges(typ, s='', o='', test_mode = False):
# if test_mode:
# test = " limit 10"
# else:
# test = ""
# """ typ = edge label. o, s = subject and object labels. These hould be pre prepended with ':'"""
# statements = ["MATCH (n%s)-[r:%s]->(m%s) RETURN n.short_form, r.label, m.short_form %s" % (s, typ, o, test)]
# r = nc.commit_list(statements)
# triples = [x['row'] for x in r[0]['data']]
# statements = []
# # Iterate over, making named edges for labels (sub space for _)
# print("Processing %d triples" % len(triples))
# for t in triples:
# subj = t[0]
import sys
from uk.ac.ebi.vfb.neo4j.flybase2neo.fb_tools import dict_cursor, get_fb_conn
from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect
import re
"""Populate pub data. Should be run as a final step, once all content added."""
## TODO: Add pub relationships (P3)
base_uri = sys.argv[1]
usr = sys.argv[2]
pwd = sys.argv[3]
nc = neo4j_connect(base_uri, usr, pwd)
# Pull all pub FBrfs from graph
statements = ['MATCH (p:pub) RETURN DISTINCT p.FlyBase']
pub_list_results = nc.commit_list(statements)
pub_list = [str(x['row'][0]) for x in pub_list_results[0]['data']
] # Parsing returned Json for results.
c = get_fb_conn()
cursor = c.cursor()
def gen_micro_ref_from_miniref(miniref):
# Use regex to truncate after year, remove brackets.
return
def gen_micro_ref_from_db():
# Merge strategy 1: # Merge requires all labels on nodes retrieved from KB to match those those on nodes with same key in prod # Scenario 1: Node loaded from OWL, has extra labels in KB # KB Load will fail. # Merge strategy 2 # If loading edge to prod loads nodes through merge => duplicate nodes. # Solution is to align node labels from KB -> Prod prior to edge loading. # But how to do this safely? Can't just match on iri! # Two options: # - Implement a label mover that works with match statements. Run this with equivalent queries before # running edge loader # - Break up match statement on edge mover into s, r, o chunks. Use the s and o chunks to specify edges to move. # Limitation of this approach is that only simple match statements or possible # - with just a triple and no 'where' clause. kb = neo4j_connect(sys.argv[1], sys.argv[2], sys.argv[3]) prod = neo4j_connect(sys.argv[4], sys.argv[5], sys.argv[6]) ni = node_importer(sys.argv[4], sys.argv[5], sys.argv[6]) ni.add_default_constraint_set(['DataSet', 'Site', 'License', 'Individual']) ncm = neo4jContentMover(kb, prod) # Labels may not match on nodes already loaded. The following finds all Individuals in KB that are in Prod, # checks for differences in labels, and adds them to the equivalent node in Prod if they match. ncm.move_node_labels(match="MATCH (n:Individual) ", node_key='iri') ## move channels and directly associated edges ## Move all instance of channel:
from uk.ac.ebi.vfb.neo4j.neo4j_tools import neo4j_connect, results_2_dict_list
import sys
import warnings
import json
# KB tests
# Query for dataSets (would be helpful if these were tagged as image)
nc = neo4j_connect(sys.argv[1], sys.argv[2], sys.argv[3])
def query(query):
q = nc.commit_list([query])
if not q:
return False
dc = results_2_dict_list(q)
if not dc:
return False
else:
return dc
def query_ind_count(query):
q = nc.commit_list([query])
if not q:
return False
dc = results_2_dict_list(q)
if not dc:
return False
if not ('ind_count' in dc[0].keys()):
warnings.warn("Query has no ind_count")
return False
def setUp(self):
To = neo4j_connect('http://localhost:7474', 'neo4j', 'neo4j')
From = neo4j_connect('http://kb.virtualflybrain.org', 'neo4j', 'neo4j')
self.ncm = neo4jContentMover(From=From, To=To)
def setUp(self):
self.nc = neo4j_connect('http://localhost:7474', 'neo4j', 'neo4j')
def _init(self, endpoint, usr, pwd):
self.conn = get_fb_conn()
self.nc = neo4j_connect(endpoint, usr, pwd)