def train_ner(output_dir: str, data_path: str, run_test: bool = None, model: str = None, n_iter: int = 100, label_granularity: int = None): if label_granularity is not None: umls_tree = construct_umls_tree_from_tsv( "data/umls_semantic_type_tree.tsv") label_mapping = umls_tree.get_collapsed_type_id_map_at_level( label_granularity) if label_granularity == 0: span_only = True else: label_mapping = None span_only = False train_data, dev_data, test_data = read_full_med_mentions( data_path, label_mapping, span_only) os.makedirs(output_dir, exist_ok=True) if run_test: nlp = spacy.load(model) print("Loaded model '%s'" % model) evaluate_ner(nlp, dev_data, dump_path=os.path.join(output_dir, "dev_metrics.json")) evaluate_ner(nlp, test_data, dump_path=os.path.join(output_dir, "test_metrics.json")) else: train(model, train_data, dev_data, output_dir, n_iter)
def __init__( self, file_path: str = DEFAULT_UMLS_PATH, types_file_path: str = DEFAULT_UMLS_TYPES_PATH, ): super().__init__(file_path) self.semantic_type_tree: UmlsSemanticTypeTree = construct_umls_tree_from_tsv( types_file_path)
def __init__(self, file_path: str = DEFAULT_UMLS_PATH, types_file_path: str = DEFAULT_UMLS_TYPES_PATH): raw = json.load(open(cached_path(file_path))) alias_to_cuis: Dict[str, Set[str]] = defaultdict(set) self.cui_to_entity: Dict[str, UmlsEntity] = {} for concept in raw: unique_aliases = set(concept["aliases"]) unique_aliases.add(concept["canonical_name"]) for alias in unique_aliases: alias_to_cuis[alias].add(concept["concept_id"]) self.cui_to_entity[concept["concept_id"]] = UmlsEntity(**concept) self.alias_to_cuis: Dict[str, Set[str]] = {**alias_to_cuis} self.semantic_type_tree: UmlsSemanticTypeTree = construct_umls_tree_from_tsv( types_file_path)
""" utils for reading MedMentions original format adapted from scispacy: https://github.com/allenai/scispacy """ from typing import NamedTuple, List, Iterator, Dict, Tuple import tarfile import atexit import os import shutil import tempfile from scispacy.file_cache import cached_path from scispacy.umls_semantic_type_tree import construct_umls_tree_from_tsv umls_tree = construct_umls_tree_from_tsv("data/umls_semantic_type_tree.tsv") class MedMentionEntity(NamedTuple): start: int end: int mention_text: str mention_type: str umls_id: str class MedMentionExample(NamedTuple): title: str abstract: str text: str pubmed_id: str entities: List[MedMentionEntity]
""" This script expects you've followed the instructions in https://github.com/chb/py-umls to install UMLS. """ import json from collections import Counter import sqlite3 from scispacy.umls_semantic_type_tree import construct_umls_tree_from_tsv umls_tree = construct_umls_tree_from_tsv('data/umls_semantic_type_tree.tsv') # change to your location umls_db_path = '/home/dan/projects/py-umls/databases/umls.db' # change to your location conn = sqlite3.connect(umls_db_path) c = conn.cursor() cui_data = {} source_counter = Counter() def_mismatches = set() st21pv_sources = set(['CPT', 'FMA', 'GO', 'HGNC', 'HPO', 'ICD10', 'ICD10CM', 'ICD9CM', 'MDR', 'MSH', 'MTH', 'NCBI', 'NCI', 'NDDF', 'NDFRT', 'OMIM', 'RXNORM', 'SNOMEDCT_US']) st21pv_types = set(['T005', 'T007', 'T017', 'T022', 'T031', 'T033', 'T037', 'T038', 'T058', 'T062', 'T074', 'T082', 'T091', 'T092', 'T097', 'T098', 'T103', 'T168', 'T170', 'T201', 'T204'])
def setUp(self): super().setUp() self.tree = construct_umls_tree_from_tsv( "tests/fixtures/test_umls_tree.tsv")