def route_dynamic(path): args = dict(request.args) if 'direction' in args: direction = args.pop('direction') else: direction = 'OUTGOING' # should always be outgoing here since we can't specify? if 'format' in args: format_ = args.pop('format') else: format_ = None try: j = sgd.dispatch(path, **args) except rHTTPError as e: log.exception(e) abort(e.response.status_code) # DO NOT PASS ALONG THE MESSAGE except ValueError as e: log.exception(e) abort(404) if j is None or 'edges' not in j or not j['edges']: log.error(pformat(j)) log.debug(sgd._last_url) return abort(400) prov = [ hfn.titletag(f'Dynamic query result for {path}'), f'<meta name="date" content="{UTCNOWISO()}">', f'<link rel="http://www.w3.org/ns/prov#wasGeneratedBy" href="{wgb}">', '<meta name="representation" content="SciGraph">', f'<link rel="http://www.w3.org/ns/prov#wasDerivedFrom" href="{sgd._last_url}">' ] kwargs = {'json': cleanBad(j), 'html_head': prov} tree, extras = creatTree(*Query(None, None, direction, None), **kwargs) #print(extras.hierarhcy) #print(tree) if format_ is not None: if format_ == 'table': #breakpoint() def nowrap(class_, tag=''): return (f'{tag}.{class_}' '{ white-space: nowrap; }') ots = [ OntTerm(n) for n in flatten_tree(extras.hierarchy) if 'CYCLE' not in n ] #rows = [[ot.label, ot.asId().atag(), ot.definition] for ot in ots] rows = [[ot.label, hfn.atag(ot.iri, ot.curie), ot.definition] for ot in ots] return hfn.htmldoc(hfn.render_table(rows, 'label', 'curie', 'definition'), styles=(hfn.table_style, nowrap('col-label', 'td'))) return hfn.htmldoc(extras.html, other=prov, styles=hfn.tree_styles)
def connectivity_query(relationship=None, start=None, end=None): j = sgd.dispatch('/dynamic/shortestSimple?' 'start_id={start.quoted}&' 'end_id={end.quoted}&' 'relationship={relationship}') kwargs['json'] = j tree, extras = creatTree(*Query(root, pred, direction, depth), **kwargs) return htmldoc(extras.html, styles=hfn.tree_styles)
def import_tree(graph): mg = makeGraph('', graph=graph) mg.add_known_namespaces('owl', 'obo', 'dc', 'dcterms', 'dctypes', 'skos', 'NIFTTL') j = mg.make_scigraph_json('owl:imports', direct=True) t, te = creatTree(*Query('NIFTTL:nif.ttl', 'owl:imports', 'OUTGOING', 30), json=j, prefixes=mg.namespaces) #print(t) return t, te
def print_trees(graph, bridge): PPO = 'ro:proper_part_of' HPP = 'ro:has_proper_part' hpp = HPP.replace('ro:', graph.namespaces['ro']) ppo = PPO.replace('ro:', graph.namespaces['ro']) a, b = creatTree( *Query( tc.red('birnlex_796'), HPP, 'OUTGOING', 10 ), # FIXME seems to be a last one wins bug here with birnlex_796 vs NIFGA:birnlex_796 depending on the has seed... json=graph.make_scigraph_json(HPP)) c, d = creatTree(*Query('NIFGA:birnlex_796', hpp, 'OUTGOING', 10), graph=sgg) j = bridge.make_scigraph_json( HPP) # issue https://github.com/RDFLib/rdflib/pull/661 e, f = creatTree(*Query('UBERON:0000955', HPP, 'OUTGOING', 10), json=j) k_, l_ = creatTree(*Query('NIFGA:nlx_anat_101177', ppo, 'INCOMING', 10), graph=sgg) merge = dict(d[-1]) # full tree with ppo converted to hpp merge['nodes'].extend(l_[-1]['nodes']) merge['edges'].extend([{ 'sub': e['obj'], 'pred': hpp, 'obj': e['sub'] } for e in l_[-1]['edges']]) m_, n_ = creatTree(*Query('NIFGA:birnlex_796', hpp, 'OUTGOING', 10), json=merge) print('nifga dep') print(a) print('nifga live') print(c) print('new bridge') print(e) print('nifga total (both directions)') print(m_) print('nifga white matter') print(k_) return a, b, c, d, e, f, k_, l_, m_, n_
def import_tree(graph, ontologies, **kwargs): for ontology in ontologies: thisfile = Path(ontology).name print(thisfile) OntCuries.populate(graph) j = graph.asOboGraph('owl:imports', restriction=False) try: t, te = creatTree(*Query(f'NIFTTL:{thisfile}', 'owl:imports', 'OUTGOING', 30), json=j, prefixes=dict(graph.namespace_manager), **kwargs) #print(t) yield t, te except KeyError: print(tc.red('WARNING:'), 'could not find', ontology, 'in import chain') # TODO zap onts w/o imports
def make_neurolex_graph(): # neurolex test stuff nlxpref = {'ilx': 'http://uri.interlex.org/base/'} nlxpref.update(NIFPREFIXES) neurolex = makeGraph('neurolex-temp', nlxpref) neurolex.g.parse('/tmp/neurolex_basic.ttl', format='turtle') ILXPO = 'ilx:partOf' nj = neurolex.make_scigraph_json(ILXPO) g_, h = creatTree(*Query('NIFGA:birnlex_796', ILXPO, 'INCOMING', 10), json=nj) i_, j_ = creatTree(*Query('NIFGA:nlx_412', ILXPO, 'INCOMING', 10), json=nj) brht = sorted(set(flatten(h[0], []))) wmht = sorted(set(flatten(j_[0], []))) ufixedrb = { 'NIFGA:' + k.split(':')[1]: v for k, v in u_replaced_by.items() } b_nlx_replaced_by = new_replaced_by(brht, ufixedrb) w_nlx_replaced_by = new_replaced_by(wmht, ufixedrb) additional_edges = defaultdict( list) # TODO this could be fun for the future but is a nightmare atm for edge in h[-1]['edges'] + j_[-1]['edges']: additional_edges[edge['sub']] = edge additional_edges[edge['obj']] = edge #filter out bad edges becase we are lazy additional_edges = { k: v for k, v in additional_edges.items() if k in b_nlx_replaced_by or k in w_nlx_replaced_by } print('neurolex tree') # computed above print(g_) print(i_) return additional_edges
def import_tree(graph, ontologies, **kwargs): for ontology in ontologies: thisfile = Path(ontology).name print(thisfile) mg = makeGraph('', graph=graph) mg.add_known_namespaces('owl', 'obo', 'dc', 'dcterms', 'dctypes', 'skos', 'NIFTTL') j = mg.make_scigraph_json('owl:imports', direct=True) try: t, te = creatTree(*Query(f'NIFTTL:{thisfile}', 'owl:imports', 'OUTGOING', 30), json=j, prefixes=mg.namespaces, **kwargs) #print(t) yield t, te except KeyError: print(tc.red('WARNING:'), 'could not find', ontology, 'in import chain') # TODO zap onts w/o imports
def route_sparc_dynamic(path): args = dict(request.args) if 'direction' in args: direction = args.pop('direction') else: direction = 'OUTGOING' # should always be outgoing here since we can't specify? if 'format' in args: format_ = args.pop('format') else: format_ = None j = data_sgd.dispatch(path, **args) #breakpoint() if not j['edges']: log.error(pprint(j)) return abort(400) kwargs = {'json': j} tree, extras = creatTree(*Query(None, None, direction, None), **kwargs) #print(extras.hierarhcy) print(tree) if format_ is not None: if format_ == 'table': #breakpoint() def nowrap(class_, tag=''): return (f'{tag}.{class_}' '{ white-space: nowrap; }') ots = [ OntTerm(n) for n in flatten_tree(extras.hierarchy) if 'CYCLE' not in n ] #rows = [[ot.label, ot.asId().atag(), ot.definition] for ot in ots] rows = [[ot.label, hfn.atag(ot.iri, ot.curie), ot.definition] for ot in ots] return htmldoc(hfn.render_table(rows, 'label', 'curie', 'definition'), styles=(hfn.table_style, nowrap('col-label', 'td'))) return htmldoc(extras.html, styles=hfn.tree_styles)
def main(): args = docopt(__doc__, version='0.1') outloc = args['--output-location'] filename = os.path.splitext(os.path.basename(args['<file>']))[0] mgraph = makeGraph(filename, prefixes=uPREFIXES, writeloc=outloc) if args['workflow']: w = WorkflowMapping(args['<file>']) [mgraph.g.add(t) for t in w.triples] w.post_graph(mgraph.g) elif args['paper']: w = PaperIdMapping(args['<file>']) [mgraph.g.add(t) for t in w.triples] w.post_graph(mgraph.g) elif args['methods']: parser = etree.XMLParser(remove_blank_text=True) e = etree.parse(args['<file>'], parser) #graph = e.find(abv['graph']) nodes = xpath(e, '//' + abv['node']) edges = xpath(e, '//' + abv['edge']) node_dict = {} for node in nodes: # slow but who cares id_ = node.get('id') #label = xpath(node, '//'+abv['NodeLabel'])[0].text idstr = '[@id="%s"]//' % id_ label = xpath(e, '//' + abv['node'] + idstr + abv['NodeLabel'])[0].text targets = [] node_dict['FIXME:' + id_] = label, targets edge_dict = {} edge_types = set() for edge in edges: id_ = edge.get('id') #print(id_) idstr = '[@id="%s"]//' % id_ source = 'FIXME:' + edge.get('source') target = 'FIXME:' + edge.get('target') out = xpath(edge, '//' + abv['edge'] + idstr + abv['EdgeLabel']) edge_type = out[0].text if out else None #print(edge_type) edge_dict[id_] = source, target, edge_replace(edge_type) edge_types.add(edge_type) for et in set(edge_to_ttl.values()): if et != 'SKIP': mgraph.add_op(et) for eid, (source, target, edge_type) in edge_dict.items(): node_dict[source][1].append((edge_type, target)) #print(source, edge_type, target) if edge_type == 'SKIP': mgraph.add_trip(source, 'rdf:type', 'owl:Class') elif edge_type is not None: mgraph.add_class(source) mgraph.add_class(target) try: if edge_type == 'rdfs:subClassOf': mgraph.add_trip(source, edge_type, target) else: mgraph.add_hierarchy(target, edge_type, source) except ValueError as e: raise ValueError(f'{source} {edge_type} {target}') from e label = node_dict[source][0] if '(id' in label: label, rest = label.split('(id') id_, rest = rest.split(')', 1) mgraph.add_trip(source, 'FIXME:REPLACEID', id_) label = label.strip() + rest.strip() if '(syns' in label: label, rest = label.split('(syns') syns, rest = rest.split(')', 1) if ',' in syns: syns = [ mgraph.add_trip(source, 'NIFRID:synonym', s.strip()) for s in syns.split(',') if s ] #FIXME else: syns = [ mgraph.add_trip(source, 'NIFRID:synonym', s) for s in syns.split(' ') if s ] #FIXME label = label.strip() + rest.strip() if '(note' in label: while '(note' in label: label, rest = label.split('(note', 1) note, rest = rest.split(')', 1) mgraph.add_trip(source, 'rdfs:comment', note) label = label.strip() + rest.strip() if '(def' in label: label, rest = label.split('(def') def_, rest = rest.split(')', 1) def_ = def_.replace('\n', ' ') mgraph.add_trip(source, 'NIFRID:definition', def_.strip()) label = label.strip() + rest if '#FIXME' in label: label, note = label.split('#FIXME') label = label.replace('\n', '').strip() note = note.replace('\n', ' ').strip() mgraph.add_trip(source, 'rdfs:comment', note) if args['methods']: clabel = label.capitalize() else: clabel = label mgraph.add_trip(source, 'rdfs:label', clabel) Query = namedtuple('Query', ['root', 'relationshipType', 'direction', 'depth']) json = mgraph.make_scigraph_json('rdfs:subClassOf', direct=True) t, te = creatTree(*Query('FIXME:n0', 'rdfs:subClassOf', 'INCOMING', 20), json=json) # methods t, te = creatTree(*Query('FIXME:n236', 'rdfs:subClassOf', 'INCOMING', 20), json=json) # techniques print(t) with open(os.path.join(outloc, filename + '.txt'), 'wt') as f: f.write(str(t)) with open(os.path.join(outloc, filename + '.html'), 'wt') as f: f.write(te.html) out_graph = cull_prefixes(mgraph.g, prefixes={ **dict(workflow=workflow, RRIDCUR=RRIDCUR), **uPREFIXES }) out_graph.filename = mgraph.filename out_graph.write()
yield [curie, labels, synonyms, parents] #globals()['for_burak'] = for_burak records = {c: [l, s, p] for c, l, s, p in for_burak(ng_) if l or s} with open( os.path.expanduser( '~/files/ontology-classes-with-labels-synonyms-parents.json'), 'wt') as f: json.dump(records, f, sort_keys=True, indent=2) mg.add_known_namespace('NIFTTL') j = mg.make_scigraph_json('owl:imports', direct=True) #asdf = sorted(set(_ for t in graph for _ in t if type(_) == rdflib.URIRef)) # this snags a bunch of other URIs #asdf = sorted(set(_ for _ in graph.subjects() if type(_) != rdflib.BNode)) asdf = set(_ for t in graph.subject_predicates() for _ in t if type(_) == rdflib.URIRef) prefs = set( _.rsplit('#', 1)[0] + '#' if '#' in _ else (_.rsplit('_', 1)[0] + '_' if '_' in _ else _.rsplit('/', 1)[0] + '/') for _ in asdf) nots = set(_ for _ in prefs if _ not in vals) sos = set(prefs) - set(nots) print(len(prefs)) t, te = creatTree(*Query('NIFTTL:nif.ttl', 'owl:imports', 'OUTGOING', 30), json=j) embed() print(t)
def render(pred, root, direction=None, depth=10, local_filepath=None, branch='master', restriction=False, wgb='FIXME', local=False, verbose=False, flatten=False,): kwargs = {'local':local, 'verbose':verbose} prov = makeProv(pred, root, wgb) if local_filepath is not None: github_link = ('https://github.com/SciCrunch/NIF-Ontology/raw/' f'{branch}/{local_filepath}') prov.append('<link rel="http://www.w3.org/ns/prov#wasDerivedFrom" ' f'href="{github_link}">') graph = graphFromGithub(github_link, verbose) qname = graph.namespace_manager._qhrm # FIXME labels_index = {qname(s):str(o) for s, o in graph[:rdfs.label:]} if pred == 'subClassOf': pred = 'rdfs:subClassOf' # FIXME qname properly? elif pred == 'subPropertyOf': pred = 'rdfs:subPropertyOf' try: kwargs['json'] = graph.asOboGraph(pred, restriction=restriction) kwargs['prefixes'] = {k:str(v) for k, v in graph.namespace_manager} except KeyError as e: if verbose: log.error(str(e)) return abort(422, 'Unknown predicate.') else: kwargs['graph'] = sgg # FIXME this does not work for a generic scigraph load ... # and it should not be calculated every time anyway! # oh look, here we are needed a class again if False: versionIRI = [ e['obj'] for e in sgg.getNeighbors('http://ontology.neuinfo.org/' 'NIF/ttl/nif.ttl')['edges'] if e['pred'] == 'versionIRI'][0] #print(versionIRI) prov.append('<link rel="http://www.w3.org/ns/prov#wasDerivedFrom" ' f'href="{versionIRI}">') # FIXME wrong and wont resolve prov.append('<meta name="representation" content="SciGraph">') # FIXME :/ kwargs['html_head'] = prov try: if root.startswith('http'): # FIXME this codepath is completely busted? if 'prefixes' in kwargs: rec = None for k, v in kwargs.items(): if root.startswith(v): rec = k + 'r:' + root.strip(v) # FIXME what?! break if rec is None: raise KeyError('no prefix found for {root}') else: rec = sgv.findById(root) if 'curie' in rec: root_curie = rec['curie'] # FIXME https://github.com/SciGraph/SciGraph/issues/268 if not root_curie.endswith(':') and '/' not in root_curie: root = root_curie else: kwargs['curie'] = root_curie elif 'prefixes' not in kwargs and root.endswith(':'): kwargs['curie'] = root root = sgc._curies[root.rstrip(':')] # also 268 tree, extras = creatTree(*Query(root, pred, direction, depth), **kwargs) dematerialize(list(tree.keys())[0], tree) if flatten: if local_filepath is not None: def safe_find(n): return {'labels':[labels_index[n]], 'deprecated': False # FIXME inacurate } else: def safe_find(n): # FIXME scigraph bug if n.endswith(':'): n = sgc._curies[n.rstrip(':')] elif '/' in n: prefix, suffix = n.split(':') iriprefix = sgc._curies[prefix] n = iriprefix + suffix return sgv.findById(n) out = set(n for n in flatten_tree(extras.hierarchy)) try: lrecs = Async()(deferred(safe_find)(n) for n in out) except RuntimeError: asyncio.set_event_loop(current_app.config['loop']) lrecs = Async()(deferred(safe_find)(n) for n in out) rows = sorted(((r['labels'][0] if r['labels'] else '') + ',' + n for r, n in zip(lrecs, out) # FIXME still stuff wrong, but better for non cache case if not r['deprecated']), key=lambda lid: lid.lower()) return '\n'.join(rows), 200, {'Content-Type':'text/plain;charset=utf-8'} else: return hfn.htmldoc(extras.html, other=prov, styles=hfn.tree_styles) except (KeyError, TypeError) as e: if verbose: log.error(f'{type(e)} {e}') if sgg.getNode(root): # FIXME distinguish these cases... message = 'Unknown predicate or no results.' elif 'json' in kwargs: message = 'Unknown root.' r = graph.namespace_manager.expand(root) for s in graph.subjects(): if r == s: message = ('No results. ' 'You are querying a ttl file directly, ' 'did you remember to set ?restriction=true?') break else: message = 'Unknown root.' return abort(422, message)
def sparc_dynamic(data_sgd, data_sgc, path, wgb, process=lambda coll, blob: blob): args = dict(request.args) if 'direction' in args: direction = args.pop('direction') else: direction = 'OUTGOING' # should always be outgoing here since we can't specify? if 'format' in args: format_ = args.pop('format') else: format_ = None if 'apinat' in path: # FIXME bad hardcoded hack _old_get = data_sgd._get try: data_sgd._get = data_sgd._normal_get j = data_sgd.dispatch(path, **args) except ValueError as e: log.exception(e) abort(404) except rHTTPError as e: log.exception(e) abort(e.response.status_code) # DO NOT PASS ALONG THE MESSAGE finally: data_sgd._get = _old_get else: try: j = data_sgd.dispatch(path, **args) except rHTTPError as e: log.exception(e) abort(e.response.status_code) # DO NOT PASS ALONG THE MESSAGE except ValueError as e: log.exception(e) abort(404) j = process(collapse_apinat, j) if j is None or 'edges' not in j: log.error(pformat(j)) return abort(400) elif not j['edges']: return node_list(j['nodes']) # FIXME ... really should error? if path.endswith('housing-lyphs'): # FIXME hack root = 'NLX:154731' #direction = 'INCOMING' else: root = None prov = [ hfn.titletag(f'Dynamic query result for {path}'), f'<meta name="date" content="{UTCNOWISO()}">', f'<link rel="http://www.w3.org/ns/prov#wasGeneratedBy" href="{wgb}">', '<meta name="representation" content="SciGraph">', ('<link rel="http://www.w3.org/ns/prov#wasDerivedFrom" ' f'href="{data_sgd._last_url}">')] kwargs = {'json': cleanBad(j), 'html_head': prov, 'prefixes': data_sgc.getCuries(), # FIXME efficiency } tree, extras = creatTree(*Query(root, None, direction, None), **kwargs) #print(extras.hierarhcy) #print(tree) if format_ is not None: if format_ == 'table': #breakpoint() def nowrap(class_, tag=''): return (f'{tag}.{class_}' '{ white-space: nowrap; }') ots = [OntTerm(n) for n in flatten_tree(extras.hierarchy) if 'CYCLE' not in n] #rows = [[ot.label, ot.asId().atag(), ot.definition] for ot in ots] rows = [[ot.label, hfn.atag(ot.iri, ot.curie), ot.definition] for ot in ots] return hfn.htmldoc(hfn.render_table(rows, 'label', 'curie', 'definition'), styles=(hfn.table_style, nowrap('col-label', 'td'))) return hfn.htmldoc(extras.html, other=prov, styles=hfn.tree_styles)