def create_annotations(article_uri, content): annotated_doc = AnnoDoc(content) for annotator in annotators: annotated_doc.add_tier(annotator) def get_span_uri(span): h = hashlib.md5() h.update(article_uri) h.update(str(span.start) + ':' + str(span.end)) return "http://www.eha.io/types/annotation/annie/span/" + str( h.hexdigest()) for tier_name in [ 'geonames', 'diseases', 'hosts', 'modes', 'pathogens', 'symptoms' ]: tier = annotated_doc.tiers[tier_name] update_query = make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix eha: <http://www.eha.io/types/> prefix rdf: <http://www.w3.org/2000/01/rdf-schema#> prefix dc: <http://purl.org/dc/terms/> {% for span in spans %} INSERT DATA { <{{get_span_uri(span)}}> anno:annotator eha:annie {% if span.geoname %} ; rdf:type eha:geoname_annotation ; anno:geoname <http://sws.geonames.org/{{span.geoname.geonameid}}> {% else %} ; rdf:type eha:keyword_annotation ; anno:category "{{tier_name}}" {% endif %} ; anno:label "{{span.label | escape}}" ; anno:source_doc <{{source_doc}}> ; anno:start {{span.start}} ; anno:end {{span.end}} ; anno:selected-text "{{span.text | escape}}" } ; {% if tier_name == "diseases" %} INSERT DATA { {% for entity_uri in resolve_keyword(span.label) %} <{{entity_uri}}> dc:relation <{{get_span_uri(span)}}> . {% endfor %} } ; {% endif %} {% endfor %} INSERT DATA { <{{source_doc}}> anno:annotated_by eha:annie_1 } """).render(get_span_uri=get_span_uri, resolve_keyword=resolve_keyword, source_doc=article_uri, tier_name=tier_name, spans=tier.spans) sparql_utils.update(update_query)
def create_annotations(article_uri, content): annotated_doc = AnnoDoc(content) for annotator in annotators: annotated_doc.add_tier(annotator) def get_span_uri(span): h = hashlib.md5() h.update(article_uri) h.update(str(span.start) + ':' + str(span.end)) return "http://www.eha.io/types/annotation/annie/span/" + str(h.hexdigest()) for tier_name in ['geonames', 'diseases', 'hosts', 'modes', 'pathogens', 'symptoms']: tier = annotated_doc.tiers[tier_name] update_query = make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix eha: <http://www.eha.io/types/> prefix rdf: <http://www.w3.org/2000/01/rdf-schema#> prefix dc: <http://purl.org/dc/terms/> {% for span in spans %} INSERT DATA { <{{get_span_uri(span)}}> anno:annotator eha:annie {% if span.geoname %} ; rdf:type eha:geoname_annotation ; anno:geoname <http://sws.geonames.org/{{span.geoname.geonameid}}> {% else %} ; rdf:type eha:keyword_annotation ; anno:category "{{tier_name}}" {% endif %} ; anno:label "{{span.label | escape}}" ; anno:source_doc <{{source_doc}}> ; anno:start {{span.start}} ; anno:end {{span.end}} ; anno:selected-text "{{span.text | escape}}" } ; {% if tier_name == "diseases" %} INSERT DATA { {% for entity_uri in resolve_keyword(span.label) %} <{{entity_uri}}> dc:relation <{{get_span_uri(span)}}> . {% endfor %} } ; {% endif %} {% endfor %} INSERT DATA { <{{source_doc}}> anno:annotated_by eha:annie_1 } """).render( get_span_uri=get_span_uri, resolve_keyword=resolve_keyword, source_doc=article_uri, tier_name=tier_name, spans=tier.spans) sparql_utils.update(update_query)
def resolve_keyword(keyword): query = make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix oboInOwl: <http://www.geneontology.org/formats/oboInOwl#> prefix obo: <http://purl.obolibrary.org/obo/> prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> SELECT ?entity WHERE { BIND (obo:DOID_4 AS ?disease) ?entity rdfs:subClassOf* ?disease . ?entity oboInOwl:hasNarrowSynonym|oboInOwl:hasRelatedSynonym|oboInOwl:hasExactSynonym|rdfs:label ?label FILTER regex(?label, "^({{keyword | escape}})$", "i") } """).render(keyword=re.escape(keyword)) resp = sparql_utils.query(query) bindings = resp.json()['results']['bindings'] if len(bindings) == 0: print("no match for", keyword.encode('ascii', 'xmlcharrefreplace')) elif len(bindings) > 1: print("multiple matches for", keyword.encode('ascii', 'xmlcharrefreplace')) print(bindings) return [binding['entity']['value'] for binding in bindings]
def print_result(result): for binding in result.json()['results']['bindings']: for key, value in binding.items(): raw_val = value['value'] print "[" + key + "]" # Check for the delimiter used to combine results in a "group by" query group. if ";;" in raw_val: print raw_val.split(";;") continue # If the value references an annotation, query it and display # the full text. if raw_val.startswith('http://www.eha.io/types/annotation/'): query = make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix dep: <http://www.eha.io/types/annotation_prop/dep/> prefix con: <http://www.eha.io/types/content/> SELECT ?phraseStart ?phraseEnd ?prepStart ?sourceText WHERE { <{{annotation_uri}}> anno:start ?phraseStart ; anno:end ?phraseEnd ; anno:source_doc/con:text ?sourceText } """).render(annotation_uri=raw_val) result = sparql_utils.query(query) bindings = result.json()['results']['bindings'] if len(bindings) == 0: print "Could not resolve source text for:" print key, raw_val for binding in bindings: text = binding['sourceText']['value'] start = int(binding['phraseStart']['value']) end = int(binding['phraseEnd']['value']) print text[start:end] else: print raw_val print "" print "~~--~~--~~"
def print_result(result): for binding in result.json()['results']['bindings']: for key, value in binding.items(): raw_val = value['value'] print "[" + key + "]" # Check for the delimiter used to combine results in a "group by" query group. if ";;" in raw_val: print raw_val.split(";;") continue # If the value references an annotation, query it and display # the full text. if raw_val.startswith('http://www.eha.io/types/annotation/'): query = make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix dep: <http://www.eha.io/types/annotation_prop/dep/> prefix con: <http://www.eha.io/types/content/> SELECT ?phraseStart ?phraseEnd ?prepStart ?sourceText WHERE { <{{annotation_uri}}> anno:start ?phraseStart ; anno:end ?phraseEnd ; anno:source_doc/con:text ?sourceText } """).render(annotation_uri=raw_val) result = sparql_utils.query(query) bindings = result.json()['results']['bindings'] if len(bindings) == 0: print "Could not resolve source text for:" print key, raw_val for binding in bindings: text = binding['sourceText']['value'] start = int(binding['phraseStart']['value']) end = int(binding['phraseEnd']['value']) print text[start:end] else: print raw_val print "" print "~~--~~--~~"
def resolve_keyword(keyword): query = make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix oboInOwl: <http://www.geneontology.org/formats/oboInOwl#> prefix obo: <http://purl.obolibrary.org/obo/> prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> SELECT ?entity WHERE { BIND (obo:DOID_4 AS ?disease) ?entity rdfs:subClassOf* ?disease . ?entity oboInOwl:hasNarrowSynonym|oboInOwl:hasRelatedSynonym|oboInOwl:hasExactSynonym|rdfs:label ?label FILTER regex(?label, "^({{keyword | escape}})$", "i") } """).render( keyword=re.escape(keyword) ) resp = sparql_utils.query(query) bindings = resp.json()['results']['bindings'] if len(bindings) == 0: print("no match for", keyword.encode('ascii', 'xmlcharrefreplace')) elif len(bindings) > 1: print("multiple matches for", keyword.encode('ascii', 'xmlcharrefreplace')) print(bindings) return [binding['entity']['value'] for binding in bindings]
import json if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--mongo_url", default='localhost') parser.add_argument("--db_name", default='t11') args = parser.parse_args() db = pymongo.MongoClient(args.mongo_url)[args.db_name] for document in db.documents.find({}): uri = "http://t11.tater.io/documents/" + document['_id'] update_query = make_template(""" prefix xsd: <http://www.w3.org/2001/XMLSchema#> prefix tater: <http://www.eha.io/types/tater/> prefix con: <http://www.eha.io/types/content/> INSERT DATA { <{{uri}}> con:text "{{doc.body | escape}}" {% for key in ['title', 'createdAt'] %} ; tater:{{key}} {{doc[key] | sparqlCast}} {% endfor %} . } """).render(uri=uri, doc=document) sparql_utils.update(update_query) print("Imported " + uri) for code in db.keywords.find({}): uri = "http://t11.tater.io/codingKeywords/" + code['_id'] update_query = make_template(""" prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> prefix tater: <http://www.eha.io/types/tater/> INSERT DATA { <{{uri}}> rdfs:label "{{code['label'] | escape}}" ; tater:header <{{header_uri}}>
def create_annotations(article_uri, content): doc = spacy_parser(content) token_to_range = {} def update_range(r1, r2): if r1 is None: return r2 if r2 is None: return r1 return [min(r1[0], r2[0]), max(r1[1], r2[1])] for token in doc: child_token = token # import pdb; pdb.set_trace() while True: token_to_range[token] = update_range( token_to_range.get(token), update_range( token_to_range.get(child_token), [token.idx, token.idx + len(token.text)])) if token.dep_ != 'ROOT': child_token = token token = token.head else: break def get_token_uri(token): h = hashlib.md5() h.update(article_uri) h.update(str(token.idx)) return "http://www.eha.io/types/annotation/spacy/" + str(h.hexdigest()) def get_pharse_uri(token): h = hashlib.md5() h.update(article_uri) start, end = token_to_range[token] assert isinstance(start, int) assert isinstance(end, int) h.update(str(start) + ':' + str(end)) return "http://www.eha.io/types/annotation/spacy/phrase/" + str(h.hexdigest()) token_inserts = [] for token in doc: token_inserts.append(make_template(""" INSERT DATA { <{{pharse_ref}}> rdf:type eha:dependent_pharse ; anno:annotator eha:spacy ; anno:source_doc <{{source_doc}}> ; anno:start {{phrase_start}} ; anno:end {{phrase_end}} ; anno:selected-text "{{phrase_text | escape}}" ; anno:root <{{token_ref}}> } ; INSERT DATA { <{{token_ref}}> rdf:label "{{root_word | escape}}" ; anno:pos "{{pos}}" {% if entity_type %} ; anno:entity_type "{{entity_type}}" {% endif %} } ; INSERT DATA { <{{parent_phrase_ref}}> dep:{{dep | replace_invalid_uri_chars}} <{{pharse_ref}}> } """).render( source_doc=article_uri, phrase_start=token_to_range[token][0], phrase_end=token_to_range[token][1], phrase_text=doc.text[slice(*token_to_range[token])], root_word=token.text, pos=token.pos_, entity_type=token.ent_type_, token_ref=get_token_uri(token), pharse_ref=get_pharse_uri(token), parent_phrase_ref=get_pharse_uri(token.head), dep=token.dep_)) for chunk in more_itertools.chunked(token_inserts, 200): sparql_utils.update(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix dep: <http://www.eha.io/types/annotation_prop/dep/> prefix eha: <http://www.eha.io/types/> prefix rdf: <http://www.w3.org/2000/01/rdf-schema#> """ + ";".join(chunk)) sparql_utils.update(make_template(""" prefix anno: <http://www.eha.io/types/annotation_prop/> prefix eha: <http://www.eha.io/types/> INSERT DATA { <{{source_doc}}> anno:annotated_by eha:spacy_0 } """).render(source_doc=article_uri))
if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument( "--max_items", default="-1" ) args = parser.parse_args() max_items = int(args.max_items) article_query_template = make_template(""" prefix con: <http://www.eha.io/types/content/> prefix anno: <http://www.eha.io/types/annotation_prop/> prefix eha: <http://www.eha.io/types/> SELECT ?item_uri ?content WHERE { ?item_uri con:text ?content FILTER NOT EXISTS { ?item_uri anno:annotated_by eha:spacy_0 } } ORDER BY asc(?item_uri) LIMIT 100 """) items_processed = 0 while max_items < 0 or items_processed < max_items: print("Items processed: ", str(items_processed)) result = sparql_utils.query(article_query_template.render()) bindings = result.json()['results']['bindings'] if len(bindings) == 0: print("No more results") break else:
if min_date: query["promedDate"] = {"$gte": min_date} print("Number of articles to process:") print(db.posts.find(query).count()) for post in db.posts.find(query): # Create triples for post post_uri = "http://www.promedmail.org/post/" + post['promedId'] update_query = make_template(prefixes + """ INSERT DATA { <{{post_uri}}> pro:date "{{promedDate | sparqlDate}}"^^xsd:dateTime ; pro:subject_raw "{{subject.raw | escape}}" ; pro:archiveNumber "{{archiveNumber}}" {% for linkedReport in resolvedLinkedReports %} ; pro:linkedReport <{{linkedReport}}> {% endfor %} {% if feedId %} ; pro:feed_id "{{feedId}}" {% endif %} } """).render(min_date=min_date, post_uri=post_uri, resolvedLinkedReports=filter( lambda x: x, map(resolve_report, post['linkedReports'])), **post) sparql_utils.update(update_query) for idx, art in enumerate(post["articles"]): if not 'content' in art: continue # Create triples for article within the post article_uri = post_uri + "#" + str(idx) update_query = make_template(prefixes + """ INSERT DATA {
if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument( "--max_items", default="-1" ) args = parser.parse_args() max_items = int(args.max_items) query_template = make_template(""" prefix con: <http://www.eha.io/types/content/> prefix anno: <http://www.eha.io/types/annotation_prop/> prefix eha: <http://www.eha.io/types/> SELECT ?item_uri ?content WHERE { ?item_uri con:text ?content # FILTER(strstarts(str(?item_uri), "http://t11.tater.io/documents/")) FILTER NOT EXISTS { ?item_uri anno:annotated_by eha:annie_1 } } ORDER BY rand() LIMIT 100 """) items_processed = 0 while max_items < 0 or items_processed < max_items: print("Items processed: ", str(items_processed)) result = sparql_utils.query(query_template.render()) bindings = result.json()['results']['bindings'] if len(bindings) == 0: print("No more results") break
) parser.add_argument( "--db_name", default='t11' ) args = parser.parse_args() db = pymongo.MongoClient(args.mongo_url)[args.db_name] for document in db.documents.find({}): uri = "http://t11.tater.io/documents/" + document['_id'] update_query = make_template(""" prefix xsd: <http://www.w3.org/2001/XMLSchema#> prefix tater: <http://www.eha.io/types/tater/> prefix con: <http://www.eha.io/types/content/> INSERT DATA { <{{uri}}> con:text "{{doc.body | escape}}" {% for key in ['title', 'createdAt'] %} ; tater:{{key}} {{doc[key] | sparqlCast}} {% endfor %} . } """).render( uri=uri, doc=document ) sparql_utils.update(update_query) print("Imported " + uri) for code in db.keywords.find({}): uri = "http://t11.tater.io/codingKeywords/" + code['_id'] update_query = make_template(""" prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> prefix tater: <http://www.eha.io/types/tater/> INSERT DATA {
prefix xsd: <http://www.w3.org/2001/XMLSchema#> """ containment_query_template = make_template(prefixes+""" INSERT { ?p1 anno:contains ?p2 } WHERE { ?p1 anno:start ?p1start ; anno:end ?p1end ; anno:source_doc ?same_source . ?dep_rel rdf:type anno:dependency_relation . ?parent ?dep_rel ?p1 . ?p2 anno:start ?p2start ; anno:end ?p2end ; anno:source_doc ?same_source ; anno:category "diseases" . ?same_source pro:post/pro:date ?source_date . FILTER ( ?p1start <= ?p2start && ?p1end >= ?p2end ) FILTER (?p1 != ?p2) {% if min_date %} FILTER (?source_date >= "{{min_date | sparqlDate}}"^^xsd:dateTime) {% endif %} {% if max_date %} FILTER (?source_date < "{{max_date | sparqlDate}}"^^xsd:dateTime) {% endif %} } """) if __name__ == '__main__': import argparse
"$gte": min_date } print("Number of articles to process:") print(db.posts.find(query).count()) for post in db.posts.find(query): # Create triples for post post_uri = "http://www.promedmail.org/post/" + post['promedId'] update_query = make_template(prefixes+""" INSERT DATA { <{{post_uri}}> pro:date "{{promedDate | sparqlDate}}"^^xsd:dateTime ; pro:subject_raw "{{subject.raw | escape}}" ; pro:archiveNumber "{{archiveNumber}}" {% for linkedReport in resolvedLinkedReports %} ; pro:linkedReport <{{linkedReport}}> {% endfor %} {% if feedId %} ; pro:feed_id "{{feedId}}" {% endif %} } """).render( min_date=min_date, post_uri=post_uri, resolvedLinkedReports=filter(lambda x:x, map(resolve_report, post['linkedReports'])), **post) sparql_utils.update(update_query) for idx, art in enumerate(post["articles"]): if not 'content' in art: continue # Create triples for article within the post article_uri = post_uri + "#" + str(idx) update_query = make_template(prefixes+""" INSERT DATA {
if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument("--max_items", default="-1") args = parser.parse_args() max_items = int(args.max_items) query_template = make_template(""" prefix con: <http://www.eha.io/types/content/> prefix anno: <http://www.eha.io/types/annotation_prop/> prefix eha: <http://www.eha.io/types/> SELECT ?item_uri ?content WHERE { ?item_uri con:text ?content # FILTER(strstarts(str(?item_uri), "http://t11.tater.io/documents/")) FILTER NOT EXISTS { ?item_uri anno:annotated_by eha:annie_1 } } ORDER BY rand() LIMIT 100 """) items_processed = 0 while max_items < 0 or items_processed < max_items: print("Items processed: ", str(items_processed)) result = sparql_utils.query(query_template.render()) bindings = result.json()['results']['bindings'] if len(bindings) == 0: print("No more results") break
prefix xsd: <http://www.w3.org/2001/XMLSchema#> """ containment_query_template = make_template(prefixes + """ INSERT { ?p1 anno:contains ?p2 } WHERE { ?p1 anno:start ?p1start ; anno:end ?p1end ; anno:source_doc ?same_source . ?dep_rel rdf:type anno:dependency_relation . ?parent ?dep_rel ?p1 . ?p2 anno:start ?p2start ; anno:end ?p2end ; anno:source_doc ?same_source ; anno:category "diseases" . ?same_source pro:post/pro:date ?source_date . FILTER ( ?p1start <= ?p2start && ?p1end >= ?p2end ) FILTER (?p1 != ?p2) {% if min_date %} FILTER (?source_date >= "{{min_date | sparqlDate}}"^^xsd:dateTime) {% endif %} {% if max_date %} FILTER (?source_date < "{{max_date | sparqlDate}}"^^xsd:dateTime) {% endif %} } """) if __name__ == '__main__': import argparse