def get_node_curie_id_from_ontology_node_id(ontology_node_id: str, ontology: ontobio.ontol.Ontology, uri_to_curie_shortener: callable): node_curie_id = None if not ontology_node_id.startswith( 'http:') and not ontology_node_id.startswith('https:'): if not ontology_node_id.startswith('OBO:'): if not ontology_node_id.startswith('UMLS:C'): node_curie_id = ontology_node_id else: node_curie_id = CUI_PREFIX + ':' + ontology_node_id.split( 'UMLS:')[1] else: node_curie_id = uri_to_curie_shortener( prefixcommons.expand_uri(ontology_node_id)) else: node_curie_id = uri_to_curie_shortener(ontology_node_id) if node_curie_id is None: kg2_util.log_message( message="could not shorten this IRI to a CURIE", ontology_name=ontology.id, node_curie_id=ontology_node_id, output_stream=sys.stderr) node_curie_id = ontology_node_id # Ensure that all CUI CURIE IDs use the "CUI:" prefix (part of fix for issue #565) if is_cui_id(node_curie_id ) and get_prefix_from_curie_id(node_curie_id) != CUI_PREFIX: node_curie_id = CUI_PREFIX + ":" + get_local_id_from_curie_id( node_curie_id) return node_curie_id
def make_ontology_from_local_file(file_name: str): file_name_without_ext = os.path.splitext(file_name)[0] file_name_with_pickle_ext = file_name_without_ext + ".pickle" if not os.path.isfile(file_name_with_pickle_ext): # the ontology hsa not been saved as a pickle file, so we need to load it from a text file if not file_name.endswith('.json'): temp_file_name = tempfile.mkstemp( prefix=kg2_util.TEMP_FILE_PREFIX + '-')[1] + '.json' size = os.path.getsize(file_name) kg2_util.log_message(message="Reading ontology file: " + file_name + "; size: " + "{0:.2f}".format(size / 1024) + " KiB", ontology_name=None) cp = subprocess.run( ['owltools', file_name, '-o', '-f', 'json', temp_file_name]) # robot commented out because it is giving a NullPointerException on umls_semantictypes.owl # Once robot no longer gives a NullPointerException, we can use it like this: # cp = subprocess.run(['robot', 'convert', '--input', file_name, '--output', temp_file_name]) if cp.stdout is not None: kg2_util.log_message(message="OWL convert result: " + cp.stdout, ontology_name=None, output_stream=sys.stdout) if cp.stderr is not None: kg2_util.log_message(message="OWL convert result: " + cp.stderr, ontology_name=None, output_stream=sys.stderr) assert cp.returncode == 0 json_file = file_name_without_ext + ".json" shutil.move(temp_file_name, json_file) else: json_file = file_name size = os.path.getsize(json_file) kg2_util.log_message(message="Reading ontology JSON file: " + json_file + "; size: " + "{0:.2f}".format(size / 1024) + " KiB", ontology_name=None) ont_return = ontobio.ontol_factory.OntologyFactory().create( json_file, ignore_cache=True) else: size = os.path.getsize(file_name_with_pickle_ext) kg2_util.log_message("Reading ontology file: " + file_name_with_pickle_ext + "; size: " + "{0:.2f}".format(size / 1024) + " KiB", ontology_name=None) ont_return = pickle.load(open(file_name_with_pickle_ext, "rb")) return ont_return
def delete_ontobio_cache_json(file_name: str): file_name_hash = hashlib.sha256(file_name.encode()).hexdigest() temp_file_path = os.path.join("/tmp", file_name_hash) if os.path.exists(temp_file_path): try: kg2_util.log_message(message="Deleting ontobio JSON cache file: " + temp_file_path) os.remove(temp_file_path) except OSError as e: if e.errno == errno.ENOENT: kg2_util.log_message( message="Error deleting ontobio JSON cache file: " + temp_file_path) else: raise e
kg_edges_file_names = args.kgFileNewEdges test_mode = args.test output_file_name = args.outputFile[0] kg = json.load(open(kg_file_name, 'r')) kg_orphan_edges = {'edges': []} for kg_edges_file_name in kg_edges_file_names: kg_orphan_edges_new = [] ctr_edges_added = 0 kg_edges_new = json.load(open(kg_edges_file_name, 'r')) nodes_dict = {node['id']: node for node in kg['nodes']} for rel_dict in kg_edges_new['edges']: subject_curie = rel_dict['subject'] object_curie = rel_dict['object'] if subject_curie in nodes_dict and object_curie in nodes_dict: ctr_edges_added += 1 kg['edges'].append(rel_dict) else: kg_orphan_edges_new.append(rel_dict) kg_orphan_edges['edges'] += kg_orphan_edges_new kg2_util.log_message("number edges added: " + str(ctr_edges_added), ontology_name=kg_edges_file_name, output_stream=sys.stderr) kg2_util.log_message("number of orphan edges: " + str(len(kg_orphan_edges['edges'])), ontology_name=kg_edges_file_name, output_stream=sys.stderr) kg2_util.save_json(kg, output_file_name, test_mode) kg_file_orphan_edges = args.kgFileOrphanEdges if kg_file_orphan_edges is not None: kg2_util.save_json(kg_orphan_edges, kg_file_orphan_edges, test_mode)
arg_parser.add_argument('--outputFile', type=str, nargs='?', default=None) arg_parser.add_argument('kgFiles', type=str, nargs='+') return arg_parser if __name__ == '__main__': args = make_arg_parser().parse_args() kg_file_names = args.kgFiles test_mode = args.test output_file_name = args.outputFile kg_orphan_edges = {'nodes': [], 'edges': []} nodes = dict() rels = dict() for kg_file_name in kg_file_names: kg2_util.log_message("reading nodes from file", ontology_name=kg_file_name, output_stream=sys.stderr) kg_to_add = json.load(open(kg_file_name, 'r')) kg_to_add_nodes = kg_to_add['nodes'] for node in kg_to_add_nodes: node_id = node['id'] if node_id not in nodes: nodes[node_id] = node else: nodes[node_id] = kg2_util.merge_two_dicts(nodes[node_id], node) kg2_util.log_message("number of nodes added: " + str(len(kg_to_add_nodes)), ontology_name=kg_file_name, output_stream=sys.stderr) ctr_edges_added = 0 edges = []
def get_percent_decrease(num1, num2): return (num1 - num2) / num1 if __name__ == '__main__': args = get_args() previous_json = dict() current_json = dict() with open(args.previousFile, 'r') as previous_file: previous_json = json.load(previous_file) with open(args.currentFile, 'r') as current_file: current_json = json.load(current_file) previous_edge_sources = previous_json['number_of_edges_by_source'] current_edge_sources = current_json['number_of_edges_by_source'] for source in previous_edge_sources: edge_count = previous_edge_sources[source] current_edge_count = current_edge_sources.get(source, None) if current_edge_count == None or current_edge_count == 0: message = f"There are no edges from {source} in this build. \ There were {edge_count} in the previous build." kg2_util.log_message(message.replace(' ', ''), output_stream=sys.stderr) continue if get_percent_decrease(edge_count, current_edge_count) > 0.2: message = f"There was a significant drop in edges from {source} \ in this build. The count dropped from {edge_count} \ to {current_edge_count}" kg2_util.log_message(message.replace(' ', ''), output_stream=sys.stderr)
assert 'uri' in node_dict iri = node_dict['uri'] del node_dict['uri'] assert 'id' in node_dict id = node_dict['id'] curie_prefix = id.split(':')[0] provided_by = KG1_PROVIDED_BY_TO_KG2_PROVIDED_BY_CURIE_IDS.get( curie_prefix, None) if provided_by is None: raise Exception("unable to get provider for CURIE prefix: " + curie_prefix) iri = expand(id) if iri is None: kg2_util.log_message( message='Invalid CURIE ID that cannot be expanded to an IRI', ontology_name=kg2_util.CURIE_PREFIX_RTX_KG1 + ':' + ';' + provided_by, node_curie_id=id, output_stream=sys.stderr) symbol = node_dict.get('symbol', None) synonym_list = [] if symbol is not None: synonym_list.append(symbol) del node_dict['symbol'] category_label = node_dict['category'].replace('_', ' ') if category_label == 'molecular function': category_label = kg2_util.BIOLINK_CATEGORY_MOLECULAR_ACTIVITY elif category_label == kg2_util.BIOLINK_CATEGORY_MICRORNA and id.startswith( kg2_util.CURIE_PREFIX_NCBI_GENE + ':'): category_label = kg2_util.BIOLINK_CATEGORY_GENE synonym_list.append('Biotype:microRNA')
def get_rels_dict(nodes: dict, owl_file_information_dict_list: list, uri_to_curie_shortener: callable, map_of_node_ontology_ids_to_curie_ids: dict): rels_dict = dict() for owl_file_information_dict in owl_file_information_dict_list: ontology = owl_file_information_dict['ontology'] ontology_id = owl_file_information_dict['id'] ont_graph = ontology.get_graph() ontology_curie_id = map_of_node_ontology_ids_to_curie_ids[ontology_id] for (object_id, subject_id, predicate_dict) in ont_graph.edges(data=True): assert type(predicate_dict) == dict ontology_node = nodes.get(ontology_curie_id, None) if ontology_node is not None: ontology_update_date = ontology_node['update date'] if subject_id == OWL_BASE_CLASS or object_id == OWL_BASE_CLASS: continue if subject_id.startswith(MYSTERIOUS_BASE_NODE_ID_TO_FILTER) or \ object_id.startswith(MYSTERIOUS_BASE_NODE_ID_TO_FILTER): continue # subject_id and object_id are IDs from the original ontology objects; these may not # always be the node curie IDs (e.g., for SNOMED terms). Need to map them subject_curie_id = map_of_node_ontology_ids_to_curie_ids.get( subject_id, None) if subject_curie_id is None: kg2_util.log_message( message="ontology node ID has no curie ID in the map", ontology_name=ontology.id, node_curie_id=subject_id, output_stream=sys.stderr) continue object_curie_id = map_of_node_ontology_ids_to_curie_ids.get( object_id, None) if object_curie_id is None: kg2_util.log_message( message="ontology node ID has no curie ID in the map", ontology_name=ontology.id, node_curie_id=object_id, output_stream=sys.stderr) continue predicate_label = None edge_pred_string = predicate_dict['pred'] if subject_curie_id.startswith( 'TUI:') and object_curie_id.startswith( 'TUI:') and edge_pred_string == 'subClassOf': continue if not edge_pred_string.startswith( 'http:') and not edge_pred_string.startswith('https'): # edge_pred_string is not a URI; this is the most common case if ':' not in edge_pred_string: # edge_pred_string is not a CURIE; this is the most common subcase if edge_pred_string != 'subClassOf': predicate_curie = 'owl:' + edge_pred_string else: predicate_curie = 'rdfs:subClassOf' predicate_label = kg2_util.convert_camel_case_to_snake_case( edge_pred_string) else: # edge_pred_string is a CURIE predicate_curie = edge_pred_string predicate_node = nodes.get(predicate_curie, None) if predicate_node is not None: predicate_label = predicate_node['name'] else: # predicate has no node object defined; just pull the label out of the CURIE if edge_pred_string.startswith('OBO:'): test_curie = edge_pred_string.replace('OBO:', '').replace( '_', ':') predicate_node = nodes.get(test_curie, None) if predicate_node is None: predicate_label = edge_pred_string.split( ':')[1].split('#')[-1] else: predicate_curie = test_curie else: predicate_label = edge_pred_string predicate_iri = prefixcommons.expand_uri(predicate_curie) predicate_curie_new = uri_to_curie_shortener(predicate_iri) if predicate_curie_new is not None: predicate_curie = predicate_curie_new else: predicate_iri = edge_pred_string predicate_curie = uri_to_curie_shortener(predicate_iri) if predicate_curie is None: kg2_util.log_message(message="predicate IRI has no CURIE: " + predicate_iri, ontology_name=ontology.id, output_stream=sys.stderr) continue if subject_curie_id == object_curie_id and predicate_label == 'xref': continue if predicate_curie == 'UMLS:hasSTY': subject_node = nodes[subject_curie_id] object_node = nodes[object_curie_id] subject_description = subject_node['description'] if subject_description is None: subject_description = '' subject_node['description'] = '; '.join( list( filter(None, [ subject_description, 'UMLS Semantic Type: ' + object_node['id'] ]))) continue rel_key = make_rel_key(subject_curie_id, predicate_curie, object_curie_id, ontology_curie_id) if predicate_label is None and ':' in predicate_curie: pred_node = nodes.get(predicate_curie, None) if pred_node is not None: predicate_label = pred_node['name'] if predicate_label[0].isupper(): predicate_label = predicate_label[0].lower( ) + predicate_label[1:] assert predicate_label is not None predicate_label = predicate_label.replace(' ', '_') # Only tested on Food and Efo ontologies predicate_label = kg2_util.convert_camel_case_to_snake_case( predicate_label) if rels_dict.get(rel_key, None) is None: edge = kg2_util.make_edge(subject_curie_id, object_curie_id, predicate_iri, predicate_curie, predicate_label, ontology_id, ontology_update_date) rels_dict[rel_key] = edge for node_id, node_dict in nodes.items(): xrefs = node_dict['xrefs'] if xrefs is not None: for xref_node_id in xrefs: if xref_node_id in nodes and node_id != xref_node_id: provided_by = nodes[node_id]['provided by'] key = make_rel_key(node_id, CURIE_OBO_XREF, xref_node_id, provided_by) if rels_dict.get(key, None) is None: edge = kg2_util.make_edge(node_id, xref_node_id, IRI_OBO_XREF, CURIE_OBO_XREF, 'xref', provided_by, ontology_update_date) rels_dict[key] = edge return rels_dict
def make_nodes_dict_from_ontologies_list( ontology_info_list: list, curies_to_categories: dict, uri_to_curie_shortener: callable, category_label_to_iri_mapper: callable): ret_dict = dict() ontologies_iris_to_curies = dict() for ontology_info_dict in ontology_info_list: ontology = ontology_info_dict['ontology'] iri_of_ontology = ontology_info_dict['id'] assert iri_of_ontology is not None ontology_curie_id = uri_to_curie_shortener(iri_of_ontology) if ontology_curie_id is None or len(ontology_curie_id) == 0: ontology_curie_id = iri_of_ontology umls_sver = ontology_info_dict.get('umls-sver', None) updated_date = None if umls_sver is not None: # if you can, parse sver string into a date string updated_date = parse_umls_sver_date(umls_sver) if updated_date is None: updated_date = ontology_info_dict['file last modified timestamp'] ontology_node = kg2_util.make_node(ontology_curie_id, iri_of_ontology, ontology_info_dict['title'], 'data source', updated_date, iri_of_ontology) ontology_node['description'] = ontology_info_dict['description'] ontology_node['ontology node ids'] = [iri_of_ontology] ontology_node['xrefs'] = [] ret_dict[ontology_curie_id] = ontology_node ontologies_iris_to_curies[iri_of_ontology] = ontology_curie_id for ontology_node_id in ontology.nodes(): onto_node_dict = ontology.node(ontology_node_id) assert onto_node_dict is not None if ontology_node_id.startswith(MYSTERIOUS_BASE_NODE_ID_TO_FILTER): continue node_curie_id = get_node_curie_id_from_ontology_node_id( ontology_node_id, ontology, uri_to_curie_shortener) assert not node_curie_id.startswith('UMLS:C') # :DEBUG: iri = onto_node_dict.get('id', None) if iri is None: iri = ontology_node_id # Ensure all CUI nodes use a 'umls/cui' IRI (part of fix for #565) if is_cui_id(node_curie_id): iri = CUI_BASE_IRI + '/' + get_local_id_from_curie_id( node_curie_id) if not iri.startswith('http:') and not iri.startswith('https:'): iri = prefixcommons.expand_uri(iri) if node_curie_id.startswith( 'NCBIGene:') or node_curie_id.startswith('HGNC:'): iri = prefixcommons.expand_uri(node_curie_id) generated_iri = prefixcommons.expand_uri(node_curie_id) if generated_iri != node_curie_id: if (generated_iri.startswith('http:') or generated_iri.startswith('https:')) and \ generated_iri != iri: iri = generated_iri node_name = onto_node_dict.get('label', None) node_full_name = None [node_category_label, ontology_id_of_node_with_category ] = get_biolink_category_for_node(ontology_node_id, node_curie_id, ontology, curies_to_categories, uri_to_curie_shortener, set(), False) node_deprecated = False node_description = None node_creation_date = None node_update_date = None node_replaced_by_curie = None node_full_name = None node_publications = set() node_synonyms = set() node_xrefs = set() node_tui = None node_has_cui = False node_tui_category_label = None node_meta = onto_node_dict.get('meta', None) if node_meta is not None: node_deprecated = node_meta.get('deprecated', False) node_definition = node_meta.get('definition', None) if node_definition is not None: node_description = node_definition['val'] if node_description.startswith( 'OBSOLETE:') or node_description.startswith( 'Obsolete.'): continue node_definition_xrefs = node_definition.get('xrefs', None) if node_definition_xrefs is not None: assert type(node_definition_xrefs) == list for xref in node_definition_xrefs: xref_pub = xref_as_a_publication(xref) if xref_pub is not None: node_publications.add(xref_pub) node_synonyms_list = node_meta.get('synonyms', None) if node_synonyms_list is not None: for syn_dict in node_synonyms_list: syn_pred = syn_dict['pred'] if syn_pred == 'hasExactSynonym': node_synonyms.add(syn_dict['val']) syn_xrefs = syn_dict['xrefs'] if len(syn_xrefs) > 0: for syn_xref in syn_xrefs: syn_xref_pub = xref_as_a_publication( syn_xref) if syn_xref_pub is not None: node_publications.add(syn_xref_pub) node_xrefs_list = node_meta.get('xrefs', None) if node_xrefs_list is not None: for xref_dict in node_xrefs_list: xref_curie = xref_dict['val'] if xref_curie.startswith('MESH:'): xref_curie = xref_curie.replace('MESH:', 'MSH:') elif xref_curie.startswith('UMLS:C'): xref_curie = CUI_PREFIX + ':' + xref_curie.split( 'UMLS:')[1] node_xrefs.add(xref_curie) basic_property_values = node_meta.get('basicPropertyValues', None) if basic_property_values is not None: node_tui_list = [] for basic_property_value_dict in basic_property_values: bpv_pred = basic_property_value_dict['pred'] bpv_pred_curie = uri_to_curie_shortener(bpv_pred) if bpv_pred_curie is None: bpv_pred_curie = bpv_pred bpv_val = basic_property_value_dict['val'] if bpv_pred_curie in [ 'OIO:creation_date', 'dcterms:issued', 'HGNC:DATE_CREATED' ]: node_creation_date = bpv_val elif bpv_pred_curie == 'HGNC:DATE_LAST_MODIFIED': node_update_date = bpv_val elif bpv_pred_curie == 'IAL:0100001': assert node_deprecated node_replaced_by_uri = bpv_val node_replaced_by_curie = uri_to_curie_shortener( node_replaced_by_uri) elif bpv_pred_curie == 'UMLS:STY': # STY_BASE_IRI: node_tui_list.append(bpv_val) elif bpv_pred_curie == 'skos:prefLabel': if not node_curie_id.startswith('HGNC:'): node_name = bpv_val else: node_full_name = bpv_val if node_name is None: node_name = node_full_name elif bpv_pred_curie == 'skos:altLabel': node_synonyms.add(bpv_val) elif bpv_pred_curie == 'skos:definition': node_description = kg2_util.strip_html(bpv_val) elif bpv_pred_curie == 'HGNC:GENESYMBOL': node_name = bpv_val node_synonyms.add(bpv_val) elif bpv_pred_curie == 'UMLS:cui': node_has_cui = True if len(node_tui_list) == 1: node_tui = node_tui_list[0] node_tui_uri = posixpath.join( 'https://identifiers.org/umls/STY', node_tui) node_tui_curie = uri_to_curie_shortener(node_tui_uri) assert node_tui_curie is not None [node_tui_category_label, _] = get_biolink_category_for_node( node_tui_uri, node_tui_curie, ontology, curies_to_categories, uri_to_curie_shortener, set(), True) node_comments = node_meta.get('comments', None) if node_comments is not None: comments_str = 'COMMENTS: ' + (' // '.join(node_comments)) if node_description is not None: node_description += ' // ' + comments_str else: node_description = comments_str if node_category_label is None: node_type = onto_node_dict.get('type', None) if node_type is not None and node_type == 'PROPERTY': node_category_label = 'property' if node_category_label is None: if not node_deprecated: kg2_util.log_message("Node does not have a category", ontology.id, node_curie_id, output_stream=sys.stderr) node_category_label = 'unknown category' else: node_category_label = 'deprecated node' if node_has_cui: assert node_tui is not None or len(node_tui_list) > 0 if node_tui_category_label is None: node_tui_category_label = 'unknown category' if node_tui is not None: kg2_util.log_message( message='Node ' + ontology_node_id + ' has CUI whose TUI cannot be mapped to category: ' + node_tui) else: kg2_util.log_message( message='Node ' + ontology_node_id + ' has CUI with multiple associated TUIs: ' + ', '.join(node_tui_list)) else: if node_category_label is None: node_category_label = node_tui_category_label # override the node category label if we have a TUI node_tui_category_iri = category_label_to_iri_mapper( node_tui_category_label) ontology_curie_id = ontologies_iris_to_curies[iri_of_ontology] source_ontology_information = ret_dict.get(ontology_curie_id, None) if source_ontology_information is None: kg2_util.log_message( message= "ontology IRI has no information dictionary available", ontology_name=iri_of_ontology, output_stream=sys.stderr) assert False source_ontology_update_date = source_ontology_information[ 'update date'] if node_update_date is None: node_update_date = source_ontology_update_date if node_description is not None: node_description_xrefs_match = REGEX_XREF_END_DESCRIP.match( node_description) if node_description_xrefs_match is not None: node_description_xrefs_str = node_description_xrefs_match[ 1] node_description_xrefs_list = node_description_xrefs_str.split( ',') for node_description_xref_str in node_description_xrefs_list: node_description_xref_str = node_description_xref_str.strip( ) if ':' in node_description_xref_str: node_xrefs.add(node_description_xref_str) node_description_pubs = REGEX_PUBLICATIONS.findall( node_description) for pub_curie in node_description_pubs: node_publications.add(pub_curie) # deal with node names that are ALLCAPS if node_name is not None and node_name.isupper(): node_name = kg2_util.allcaps_to_only_first_letter_capitalized( node_name) node_dict = kg2_util.make_node(node_curie_id, iri, node_name, node_category_label, node_update_date, iri_of_ontology) node_dict['full name'] = node_full_name node_dict['description'] = node_description node_dict[ 'creation date'] = node_creation_date # slot name is not biolink standard node_dict[ 'deprecated'] = node_deprecated # slot name is not biolink standard node_dict[ 'replaced by'] = node_replaced_by_curie # slot name is not biolink standard node_dict['ontology node ids'] = [ ontology_node_id ] # slot name is not biolink standard node_dict['xrefs'] = list( node_xrefs) # slot name is not biolink standard node_dict['synonym'] = list( node_synonyms) # slot name is not biolink standard node_dict['publications'] = list(node_publications) # check if we need to make a CUI node if node_meta is not None and basic_property_values is not None: for basic_property_value_dict in basic_property_values: bpv_pred = basic_property_value_dict['pred'] bpv_pred_curie = uri_to_curie_shortener(bpv_pred) bpv_val = basic_property_value_dict['val'] if bpv_pred_curie == 'UMLS:cui': # CUI_BASE_IRI: cui_node_dict = dict(node_dict) cui_uri = bpv_pred + '/' + bpv_val cui_curie = uri_to_curie_shortener(cui_uri) assert cui_curie is not None assert not cui_curie.startswith('UMLS:C') # :DEBUG: # Skip this CUI if it's identical to the ontology node itself (happens with files created # using 'load_on_cuis' - part of fix for issue #565) if get_local_id_from_curie_id( cui_curie) == get_local_id_from_curie_id( node_curie_id): continue cui_node_dict['id'] = cui_curie cui_node_dict['iri'] = cui_uri cui_node_dict['synonym'] = [] cui_node_dict['category'] = node_tui_category_iri cui_node_dict[ 'category label'] = node_tui_category_label.replace( ' ', '_') cui_node_dict['ontology node ids'] = [] cui_node_dict['provided by'] = CUI_BASE_IRI cui_node_dict['xrefs'] = [ ] # blanking the "xrefs" here is *vital* in order to avoid issue #395 cui_node_dict_existing = ret_dict.get(cui_curie, None) if cui_node_dict_existing is not None: cui_node_dict = kg2_util.merge_two_dicts( cui_node_dict, cui_node_dict_existing) ret_dict[cui_curie] = cui_node_dict node_dict_xrefs = node_dict['xrefs'] node_dict_xrefs.append(cui_curie) node_dict['xrefs'] = list(set(node_dict_xrefs)) elif bpv_pred_curie == 'HGNC:ENTREZGENE_ID': entrez_gene_id = bpv_val entrez_node_dict = dict(node_dict) entrez_curie = 'NCBIGene:' + entrez_gene_id entrez_node_dict['id'] = entrez_curie entrez_node_dict[ 'iri'] = 'https://identifiers.org/NCBIGene/' + entrez_gene_id ret_dict[entrez_curie] = entrez_node_dict node_dict_xrefs = node_dict['xrefs'] node_dict_xrefs.append(entrez_curie) node_dict['xrefs'] = list(set(node_dict_xrefs)) if node_curie_id in ret_dict: node_dict = kg2_util.merge_two_dicts(ret_dict[node_curie_id], node_dict) ret_dict[node_curie_id] = node_dict return ret_dict
def get_biolink_category_for_node(ontology_node_id: str, node_curie_id: str, ontology: ontobio.ontol.Ontology, curies_to_categories: dict, uri_to_curie_shortener: callable, ontology_node_ids_previously_seen: set, get_node_id_of_node_with_category: bool): # if we have already looked for a category for this node, return None if ontology_node_id in ontology_node_ids_previously_seen: return [None, None] if ontology_node_id == OWL_NOTHING or node_curie_id is None: return [None, None] ontology_node_ids_previously_seen.add(ontology_node_id) curie_prefix = get_prefix_from_curie_id(node_curie_id) # Inelegant hack to ensure that TUI: nodes get mapped to "semantic type" while still enabling us # to use get_biolink_category_for_node to determine the specific semantic type of a CUI: based on its # TUI: record. Need to think about a more elegant way to do this. [SAR] if curie_prefix == 'TUI' and ontology.id.endswith('/umls/STY/'): return ['semantic type', None] if get_node_id_of_node_with_category: ret_ontology_node_id_of_node_with_category = ontology_node_id else: ret_ontology_node_id_of_node_with_category = None curies_to_categories_prefixes = curies_to_categories['prefix-mappings'] ret_category = curies_to_categories_prefixes.get(curie_prefix, None) if ret_category is None: # need to walk the ontology hierarchy until we encounter a parent term with a defined biolink category curies_to_categories_terms = curies_to_categories['term-mappings'] ret_category = curies_to_categories_terms.get(node_curie_id, None) if ret_category is None: for parent_ontology_node_id in ontology.parents( ontology_node_id, ['subClassOf']): parent_node_curie_id = get_node_curie_id_from_ontology_node_id( parent_ontology_node_id, ontology, uri_to_curie_shortener) try: [ret_category, ontology_node_id_of_node_with_category ] = get_biolink_category_for_node( parent_ontology_node_id, parent_node_curie_id, ontology, curies_to_categories, uri_to_curie_shortener, ontology_node_ids_previously_seen, get_node_id_of_node_with_category) if get_node_id_of_node_with_category and ontology_node_id_of_node_with_category is not None: ret_ontology_node_id_of_node_with_category = ontology_node_id_of_node_with_category except RecursionError: kg2_util.log_message(message="recursion error: " + ontology_node_id, ontology_name=ontology.id, node_curie_id=node_curie_id, output_stream=sys.stderr) assert False if ret_category is not None: break if ret_category is None: if node_curie_id.startswith(kg2_util.CURIE_PREFIX_ENSEMBL + ':'): curie_suffix = node_curie_id.replace( kg2_util.CURIE_PREFIX_ENSEMBL + ':', '') ensembl_match = REGEX_ENSEMBL.match(curie_suffix) if ensembl_match is not None: ensembl_match_letter = ensembl_match[1] ret_category = ENSEMBL_LETTER_TO_CATEGORY.get( ensembl_match_letter, None) if ret_category is None: kg2_util.log_message(message="unrecognized Ensembl ID: " + curie_suffix, ontology_name=ontology.id, node_curie_id=node_curie_id, output_stream=sys.stderr) return [ret_category, ret_ontology_node_id_of_node_with_category]
def make_kg2(curies_to_categories: dict, uri_to_curie_shortener: callable, map_category_label_to_iri: callable, owl_urls_and_files: tuple, output_file_name: str, test_mode: bool = False): owl_file_information_dict_list = [] # for each OWL file (or URL for an OWL file) described in the YAML config file... for ont_source_info_dict in owl_urls_and_files: if ont_source_info_dict['download']: # get the OWL file onto the local file system and get a full path to it local_file_name = kg2_util.download_file_if_not_exist_locally( ont_source_info_dict['url'], ont_source_info_dict['file']) else: local_file_name = ont_source_info_dict['file'] assert os.path.exists(ont_source_info_dict['file']) # load the OWL file dadta into an ontobio.ontol.Ontology data structure and information dictionary [ont, metadata_dict] = load_owl_file_return_ontology_and_metadata( local_file_name, ont_source_info_dict['url'], ont_source_info_dict['title']) metadata_dict['ontology'] = ont owl_file_information_dict_list.append(metadata_dict) kg2_util.log_message('Calling make_nodes_dict_from_ontologies_list') nodes_dict = make_nodes_dict_from_ontologies_list( owl_file_information_dict_list, curies_to_categories, uri_to_curie_shortener, map_category_label_to_iri) kg2_util.log_message('Calling make_map_of_node_ontology_ids_to_curie_ids') map_of_node_ontology_ids_to_curie_ids = make_map_of_node_ontology_ids_to_curie_ids( nodes_dict) kg2_util.log_message('Calling get_rels_dict') # get a dictionary of all relationships including xrefs as relationships all_rels_dict = get_rels_dict(nodes_dict, owl_file_information_dict_list, uri_to_curie_shortener, map_of_node_ontology_ids_to_curie_ids) kg2_dict = dict() kg2_dict['edges'] = [rel_dict for rel_dict in all_rels_dict.values()] kg2_util.log_message('Number of edges: ' + str(len(kg2_dict['edges']))) kg2_dict['nodes'] = list(nodes_dict.values()) kg2_util.log_message('Number of nodes: ' + str(len(kg2_dict['nodes']))) del nodes_dict # delete xrefs from all_nodes_dict for node_dict in kg2_dict['nodes']: del node_dict['xrefs'] del node_dict['ontology node ids'] kg2_util.log_message('Saving JSON file') kg2_util.save_json(kg2_dict, output_file_name, test_mode)