def test_update_from_obograph(self):
# Adding this to cope with odd issues with file_path when running python modules on different systems
p = get_file_path("uk/ac/ebi/vfb/neo4j/test/resources/vfb_ext.json")
print(p)
self.ni.update_from_obograph(file_path=p)
self.ni.commit()
result = self.ni.nc.commit_list(["MATCH (p:Property) WHERE p.iri = 'http://purl.obolibrary.org/obo/RO_0002350' RETURN p.label as label"])
dc = results_2_dict_list(result)
assert dc[0]['label'] == 'member_of'
result = self.ni.nc.commit_list(["MATCH (p:Class) WHERE p.iri = 'http://purl.obolibrary.org/obo/fbbt/vfb/VFB_10000005' RETURN p.label as label"])
dc = results_2_dict_list(result)
assert dc[0]['label'] == 'cluster'
def gen_neuron2channel(self):
"""Function to add assertions of overlap to BrainName domains. Currently works with a simple cutoff, but there is scope to modify this to at least specify a proportion of voxel size of domain."""
# Map neuron 2 neuropil overlap
# Give this a better name
# => anatomical individual -> overlaps by abbv. Now we need channels!
neurons = [d['shortFormID'] for d in self.neuron_neuropil_overlaps]
chunked_neurons = chunks(l=neurons, n=500)
s = []
rdl = []
for c in chunked_neurons:
s.append("MATCH (neuron_ind:Individual)<-[r:Related]-(neuron_channel:Individual) " \
"WHERE r.short_form = 'depicts' AND neuron_ind.short_form IN %s " \
"RETURN neuron_ind.short_form, neuron_channel.iri" % str(c))
result = self.nc.commit_list(s)
# Check if query returns
if result:
rdl.extend(results_2_dict_list(result))
# Neuron to channel lookup
neuron2channel = {}
for r in rdl:
neuron2channel[
r['neuron_ind.short_form']] = r['neuron_channel.iri']
return neuron2channel
def gen_BrainName_mapping(self):
"""Returns a dict mapping BrainName_abbv to channel."""
self.cursor.execute("SELECT b2o.BrainName_abbv, oe.shortFormID, o.baseURI " \
"FROM BrainName_to_owl b2o " \
"JOIN owl_class oe ON (oe.id=b2o.owl_class_id) " \
"JOIN ontology o ON (o.id=oe.ontology_id)")
BN_dict = {}
dc = self.cursor.fetchall()
for d in dc:
BN = d["BrainName_abbv"]
r = self.nc.commit_list(["MATCH (neuropil:Class)<-[:INSTANCEOF]-(a:Individual) " \
"<-[:Related { short_form : 'depicts'}]-(neuropil_channel:Individual)" \
"-[:in_register_with]->(bc) " \
"-[:Related { short_form : 'depicts'}]->(t:Individual)" \
"WHERE neuropil.short_form = '%s' " \
"AND t.label = 'JFRC2_template'" \
"RETURN neuropil_channel.iri " % d["shortFormID"]])
nci = results_2_dict_list(r)
if not nci:
warnings.warn("No channel mapping for %s" % BN)
elif len(nci) > 1:
warnings.warn("Multiple channel mappings for %s" % BN)
else:
BN_dict[BN] = nci[0]["neuropil_channel.iri"]
return BN_dict
def _generate_lookups(self, conf):
"""Generate :Class name:ID lookups from DB for loading by label.
Lookups are defined by standard config that specifies a config:
name:field:regex, e.g. {'part_of': {'short_form': 'FBbt_.+'}}
name should match the kwarg for which it is to be used.
"""
# This is just rolling a relation lookup...
lookup_config = conf
lookup = {}
if lookup_config:
# Add some type checking here?
for name, lcs in lookup_config.items():
lookup[name] = {}
for c in lcs:
q = "MATCH (c%s) where c.%s =~ '%s' RETURN c.label as label" \
", c.short_form as short_form" % (c.neo_label_string, c.field, c.regex)
#print(q)
rr = self.ew.nc.commit_list([q])
r = results_2_dict_list(rr)
lookup[name].update({
escape_string_for_neo(x['label']): x['short_form']
for x in r
})
return lookup
def query(query):
q = nc.commit_list([query])
if not q:
return False
dc = results_2_dict_list(q)
if not dc:
return False
else:
return dc
def get_lookup(limit_by_prefix=None):
if limit_by_prefix:
regex_string = ':.+|'.join(limit_by_prefix) + ':.+'
where = " AND a.obo_id =~ '%s' " % regex_string
else:
where = ''
nc = neo4j_connect("https://pdb.virtualflybrain.org", "neo4j", "neo4j")
lookup_query = "MATCH (a:VFB:Class) WHERE exists (a.obo_id)" + where + " RETURN a.obo_id as id, a.label as name"
q = nc.commit_list([lookup_query])
r = results_2_dict_list(q)
lookup = {x['name']: x['id'] for x in r}
#print(lookup['neuron'])
property_query = "MATCH (p:Property) WHERE exists(p.obo_id) RETURN p.obo_id as id, p.label as name"
q = nc.commit_list([property_query])
r = results_2_dict_list(q)
lookup.update({x['name']: x['id'] for x in r})
#print(lookup['neuron'])
return lookup
def query_ind_count(query):
q = nc.commit_list([query])
if not q:
return False
dc = results_2_dict_list(q)
if not dc:
return False
if not ('ind_count' in dc[0].keys()):
warnings.warn("Query has no ind_count")
return False
else:
return dc[0]['ind_count']
def testMoveNodeLabels(self):
self.ncm.To.commit_list(["CREATE (n { short_form : 'VFB_00000002' })"])
self.ncm.move_node_labels(
match="MATCH (n { short_form : 'VFB_00000002' })",
node_key='short_form')
query = self.ncm.To.commit_list([
"MATCH (n { short_form : 'VFB_00000002' })"
"RETURN labels(n) as nlab"
])
query_results = results_2_dict_list(query)
assert 'Individual' in query_results[0]['nlab']
def test_commit_from_remote_csv(self):
# Quite a minimal test...
self.nc.commit_csv(
url=
'https://neo4j.com/docs/developer-manual/3.3/csv/artists-with-headers.csv',
statement=
"MERGE (:Artist { name: line.Name, year: toInteger(line.Year)})")
q = self.nc.commit_list(
["MATCH (a:Artist { name: 'ABBA'})"
"RETURN a.year as y"])
qr = results_2_dict_list(q)
for r in qr:
assert r[
'y'] == 1992 # Note - this required cast from string to int.
def gen_simple_report(terms):
nc = neo4j_connect("https://pdb.virtualflybrain.org", "neo4j", "neo4j")
query = """MATCH (n:Class) WHERE n.iri in %s WITH n
OPTIONAL MATCH (n)-[r]->(p:pub) WHERE r.typ = 'syn'
WITH n,
COLLECT({ synonym: r.synonym, PMID: 'PMID:' + p.PMID,
miniref: p.label}) AS syns
OPTIONAL MATCH (n)-[r]-(p:pub) WHERE r.typ = 'def'
with n, syns,
collect({ PMID: 'PMID:' + p.PMID, miniref: p.label}) as pubs
OPTIONAL MATCH (n)-[:SUBCLASSOF]->(super:Class)
RETURN n.short_form as short_form, n.label as label,
n.description as description, syns, pubs,
super.label, super.short_form
""" % str(terms)
#print(query)
q = nc.commit_list([query])
return results_2_dict_list(q)
def testMoveEdges(self):
self.ncm.move_nodes(
match="MATCH (n:Individual { short_form : 'VFB_00000001' })",
key='iri')
self.ncm.move_nodes(match="MATCH (n:Class { label : 'neuron' }) ",
key='iri')
self.ncm.move_edges(
match="MATCH (s:Individual { short_form : 'VFB_00000001'})"
"-[r]-(o:Class { label : 'neuron' })",
node_key='iri')
query = self.ncm.To.commit_list([
"MATCH (s:Individual { short_form : 'VFB_00000001' })"
"-[r]-(o:Class { label : 'neuron' }) "
"RETURN type(r) as rtype"
])
query_results = results_2_dict_list(query)
for q in query_results:
assert q['rtype'] == 'INSTANCEOF'
def gen_report(server, query, report_name, column_order=None):
"""Generates a pandas dataframe with
the results of a cypher query against the
specified server.
Args:
server: server connection as [endpoint, usr, pwd]
query: cypher query
report_name: df.name
column_order: optionally specify column order in df."""
nc = neo4j_connect(*server)
print(query)
r = nc.commit_list([query])
#print(r)
dc = results_2_dict_list(r)
report = pd.DataFrame.from_records(dc)
report.replace(np.nan, '', regex=True, inplace=True)
report.name = report_name
if column_order:
out = report[column_order]
out.name = report_name
return out
else:
return report
def roll_cypher_add_syn_pub_link(sfid, s, pub_id_typ, pub_id):
"""Generates a Cypher statement that links an existing class
to a pub node ..."""
label = re.sub("'", "\'", s['name'])
return "MATCH (a:Class { short_form : \"%s\" }) " \
"MERGE (p:pub:Individual { %s : \"%s\" }) " \
"MERGE (a)-[:has_reference { typ : \"syn\", scope: \"%s\", synonym : \"%s\", cat: \"%s\" }]->(p)" \
"" % (sfid, clean_pub_id_typ(sfid, pub_id_typ), pub_id, s['scope'], label, s['type'])
nc.commit_list(["MERGE (:pub:Individual { FlyBase: 'Unattributed' })"])
q = nc.commit_list([
"MATCH (c) where c:Class or c:Individual return c.short_form as short_form, c.obo_synonym as syns, c.obo_definition_citation as def"
])
dc = results_2_dict_list(q)
statements = []
for d in dc:
if d['def']:
for cit in d['def']:
if cit:
def_cit = json.loads(cit)
for ref in def_cit['oboXrefs']:
if ref['id']:
statements.append(
roll_cypher_add_def_pub_link(
sfid=d['short_form'],
pub_id=ref['id'],
pub_id_typ=ref['database'],
))
elif d['syns']:
def make_catmaid_vfb_reports(cat_papers, cat_skids, dataset_name):
"""Make comparison with data in VFB for given sets of papers and skids in CATMAID.
Outputs a file of numbers of SKIDs per paper and a file of CATMAID SKIDs that are not in VFB."""
save_directory = "../VFB_reporting_results/CATMAID_SKID_reports/"
comparison_outfile = save_directory + dataset_name + "_comparison.tsv"
skids_outfile = save_directory + dataset_name + "_new_skids.tsv"
neuron_skids_outfile = save_directory + dataset_name + "_neuron_only_skids.tsv"
# Get table of names of catmaid datasets in VFB
pub_query = "MATCH (api:API)<-[dsxref:hasDbXref]-(ds:DataSet) " \
"WHERE api.short_form ends with '_catmaid_api' " \
"RETURN toInteger(dsxref.accession) as CATMAID_ID, ds.short_form as VFB_name"
q = nc.commit_list([pub_query])
papers = results_2_dict_list(q)
vfb_papers = pd.DataFrame.from_dict(papers)
vfb_papers = vfb_papers.set_index("CATMAID_ID")
# match up SKIDs per paper and output dict of lists of skids
skids_by_paper = {}
for paper_id in cat_papers.index: # do everything per paper
# get list of skids in CATMAID data as strings
skids_in_paper_cat = [
str(s)
for s in cat_skids[cat_skids['paper_id'] == paper_id]['skid']
]
# get skids from VFB KB and reformat to list of strings
query = "MATCH (api:API)<-[dsxref:hasDbXref]-(ds:DataSet)" \
"<-[:has_source]-(i:Individual)" \
"-[skid:hasDbXref]->(s:Site) " \
"WHERE api.short_form ends with '_catmaid_api' " \
"AND s.short_form starts with 'catmaid_' " \
"AND dsxref.accession = '" + str(paper_id) +"' WITH i, skid " \
"MATCH (i)-[:INSTANCEOF]-(c:Class) " \
"RETURN distinct skid.accession AS `r.catmaid_skeleton_ids`, c.iri"
q = nc.commit_list([query])
skids_in_paper_vfb = results_2_dict_list(q)
vfb_skid_classes_df = pd.DataFrame.from_dict(skids_in_paper_vfb)
# count skids only annotated as 'neuron'
try:
unique_skids = list(
set([x
for x in vfb_skid_classes_df['r.catmaid_skeleton_ids']]))
except KeyError: # if dataframe is empty make empty list of skids and empty df with named columns
unique_skids = []
vfb_skid_classes_df = pd.DataFrame(
columns=['c.iri', 'r.catmaid_skeleton_ids'])
neuron_iris = [
'http://purl.obolibrary.org/obo/FBbt_00005106',
'http://purl.obolibrary.org/obo/CL_0000540'
]
neuron_only_skids = []
for skid in unique_skids:
if len(vfb_skid_classes_df[vfb_skid_classes_df['r.catmaid_skeleton_ids'] == skid].index) == \
len(vfb_skid_classes_df[(vfb_skid_classes_df['r.catmaid_skeleton_ids'] == skid)
& (vfb_skid_classes_df['c.iri'].isin(neuron_iris))].index):
neuron_only_skids.append(skid)
skids_in_paper_vfb = unique_skids # take unique records only
# comparison of lists of skids
cat_not_vfb = [
s for s in skids_in_paper_cat if s not in skids_in_paper_vfb
]
vfb_not_cat = [
s for s in skids_in_paper_vfb if s not in skids_in_paper_cat
]
skids_by_paper[paper_id] = {
'skids_in_paper_cat': skids_in_paper_cat,
'skids_in_paper_vfb': skids_in_paper_vfb,
'cat_not_vfb': cat_not_vfb,
'vfb_not_cat': vfb_not_cat,
'neuron_only': neuron_only_skids
}
# make dataframe of list lengths from skid_df
skids_df = pd.DataFrame.from_dict(skids_by_paper,
orient='index') # df of lists
skids_df_count = skids_df.applymap(lambda x: len(x))
# make combined table with all info, tidy up and save as tsv
all_papers = pd.merge(cat_papers,
vfb_papers,
left_index=True,
right_index=True,
how='left',
sort=True)
all_papers = pd.concat([all_papers, skids_df_count],
join="outer",
axis=1,
sort=True)
all_papers.rename(columns={
'name': 'CATMAID_name',
'VFB_name': 'VFB_name',
'skids_in_paper_cat': 'CATMAID_SKIDs',
'skids_in_paper_vfb': 'VFB_SKIDS',
'cat_not_vfb': 'CATMAID_not_VFB',
'vfb_not_cat': 'VFB_not_CATMAID',
'neuron_only': 'neuron_only'
},
inplace=True)
all_papers.index.name = 'Paper_ID'
all_papers.to_csv(comparison_outfile, sep="\t")
# make unique set of skids in vfb
vfb_skid_list = [
skid for skidlist in skids_df['skids_in_paper_vfb']
for skid in skidlist if skid is not None
]
vfb_skid_list = list(set(vfb_skid_list))
vfb_skid_list = [int(x) for x in vfb_skid_list]
# filter cat_skids dataframe (df_skids from get_catmaid_papers) to remove rows where skid in VFB
new_skids_output = cat_skids[~cat_skids['skid'].isin(vfb_skid_list)].sort_values('skid') \
.reindex(columns=(cat_skids.columns.tolist() + ['FBbt_ID']))
new_skids_output.to_csv(skids_outfile, sep="\t",
index=False) # output file
# make unique set of skids annotated only as neuron in VFB
vfb_neuron_skid_list = [
skid for skidlist in skids_df['neuron_only'] for skid in skidlist
if skid is not None
]
vfb_neuron_skid_list = list(set(vfb_neuron_skid_list))
vfb_neuron_skid_list = [int(x) for x in vfb_neuron_skid_list]
# output file with skids only annotated as neuron
neuron_skids_output = cat_skids[cat_skids['skid'].isin(vfb_neuron_skid_list)].sort_values('paper_id') \
.reindex(columns=(cat_skids.columns.tolist() + ['FBbt_ID']))
neuron_skids_output.to_csv(neuron_skids_outfile, sep="\t", index=False)
# TERMINAL OUTPUT
new_papers = all_papers[all_papers.VFB_name.isnull()]
print(
str(len(new_papers.index)) +
" new papers in CATMAID that are not in VFB")
print(new_papers["CATMAID_name"])
print("See " + comparison_outfile + " for differences in numbers of SKIDs")
print("See " + skids_outfile + " for new SKIDs that are not yet in VFB")
print("Result: True")
return True
else:
print("Testing assertion:" + description)
print(query2)
print("Result: inds_in_datset: %d ; Compliant with pattern: %d" % (r1['ind_count'], r2['ind_count']))
# Should probably turn this into a report
bad_inds = list(set(r1['ind_list']) - set(r2['ind_list']))
file = open(dataset + ".report", 'w')
file.write(json.dumps(bad_inds))
file.close()
return False
datasets = nc.commit_list(["MATCH (ds:DataSet) RETURN ds.label"])
dc = results_2_dict_list(datasets)
return_state = True
for d in dc:
ds = d['ds.label']
dataset_status = True
print("\n")
print ("Testing: " + ds)
final_clauses = " WHERE ds.label = '%s' RETURN COUNT (DISTINCT i) as ind_count" \
", COLLECT(i.short_form) as ind_list" % ds
base_query = "MATCH (ds:DataSet)<-[:has_source]-(i:Individual)"
new_base_query = "MATCH (ds:DataSet)<-[:Annotation { short_form: 'source'}]-(i:Individual)"
if query_ind_count(base_query + final_clauses) == 0:
if query_ind_count(new_base_query + final_clauses):
base_query = new_base_query
def get_ind_id(self, i: VfbInd, context):
if i.xref_db and i.xref_acc:
# query to find id
query = "MATCH (s:Site)<-[r:hasDbXref]-(i:Individual) " \
"WHERE s.short_form = '%s' " \
"AND r.accession = '%s'" \
"RETURN i.short_form as vfb_id" % (i.xref_db, i.xref_acc)
q = self.ew.nc.commit_list([query])
if not q:
self.warn(context_name="%s individual" % context,
context=i,
message="VFB subject query fail"
) # Better make exception ?
self.stat = False # Better try except here?
else:
r = results_2_dict_list(q)
if len(r) == 1:
return r[0]['vfb_id']
elif not r:
self.warn(context_name='%s_individual' % context,
context=i,
message="Unknown xref: %s:%s"
"" % (i.xref_db, i.xref_acc))
self.stat = False
else:
self.warn(
context_name='%s_individual' % context,
context=i,
message='Multiple matches for xref: %s:%s - %s'
'' %
(i.xref_db, i.xref_acc, str([x['vfb_id'] for x in r])))
self.stat = False
elif i.id:
if i.label:
# subject label used to double-check against DB when ID provided.
# Is this the behavior we want?
# TODO - Fix for new object schema
query = """MATCH (i:Individual { short_form: '%s'})
RETURN i.label as name""" % i.id
q = self.ew.nc.commit_list([query])
if not q:
self.warn(context_name="%s individual" % context,
context=i,
message="VFB %s query fail" %
context) # Better make exception ?
self.stat = False
else:
r = results_2_dict_list(q)
if len(r) == 1:
name = r[0]['name']
elif not r:
self.warn(context_name='%s_individual' % context,
context=i,
message="Unknown individual: %s"
"" % i.id)
self.stat = False
return False
if name == i.label:
return i.id
else:
self.warn(context_name="%s individual" % context,
context=i,
message="You provided label: (%s) and ID: (%s), " \
"but in the KB this IS matches %s " \
"" % (i.label, i.id, r[0]['name']))
self.stat = False
else:
self.warn(
context_name="%s individual" % context,
context=i,
message=
"No name provided, so adding relationship to individual without name/ID crosscheck"
)
return i.id
