def get_medcat(CDB_MAP, VOCAB_MAP, CAT_MAP, project):
cdb_id = project.concept_db.id
vocab_id = project.vocab.id
cat_id = str(cdb_id) + "-" + str(vocab_id)
if cat_id in CAT_MAP:
cat = CAT_MAP[cat_id]
else:
if cdb_id in CDB_MAP:
cdb = CDB_MAP[cdb_id]
else:
cdb_path = project.concept_db.cdb_file.path
cdb = CDB()
cdb.load_dict(cdb_path)
CDB_MAP[cdb_id] = cdb
if vocab_id in VOCAB_MAP:
vocab = VOCAB_MAP[vocab_id]
else:
vocab_path = project.vocab.vocab_file.path
vocab = Vocab()
vocab.load_dict(vocab_path)
VOCAB_MAP[vocab_id] = vocab
cat = CAT(cdb=cdb, vocab=vocab)
cat.train = False
CAT_MAP[cat_id] = cat
return cat
def __init__(self, cdb=None, vocab=None, word_tokenizer=None):
self.cdb = cdb
self.w2v = None
self.vocab_path = "./vocab.dat"
if vocab is not None:
self.vocab = vocab
else:
self.vocab = Vocab()
# Build the required spacy pipeline
self.nlp = SpacyPipe(spacy_split_all, disable=['ner', 'parser', 'vectors', 'textcat'])
# Get the tokenizer
if word_tokenizer is not None:
self.tokenizer = word_tokenizer
else:
self.tokenizer = self._tok
def __init__(self, cdb, vocab=None, word_tokenizer=None):
self.cdb = cdb
self.w2v = None
if vocab is not None:
self.vocab = vocab
else:
self.vocab = Vocab()
# Build the required spacy pipeline
self.nlp = SpacyPipe(spacy_split_all, disable=['ner', 'parser', 'vectors', 'textcat'])
self.nlp.add_punct_tagger(tagger=partial(spacy_tag_punct, skip_stopwords=self.SKIP_STOPWORDS))
self.spell_checker = CustomSpellChecker(cdb_vocab=cdb.vocab, data_vocab=vocab)
self.nlp.add_spell_checker(spell_checker=self.spell_checker)
# Get the tokenizer
if word_tokenizer is not None:
self.tokenizer = word_tokenizer
else:
self.tokenizer = self._tok
def _create_cat(self):
"""
Loads MedCAT resources and creates CAT instance
"""
if os.getenv("APP_MODEL_VOCAB_PATH") is None:
raise ValueError(
"Vocabulary (env: APP_MODEL_VOCAB_PATH) not specified")
if os.getenv("APP_MODEL_CDB_PATH") is None:
raise Exception(
"Concept database (env: APP_MODEL_CDB_PATH) not specified")
# Vocabulary and Concept Database are mandatory
self.log.debug('Loading VOCAB ...')
vocab = Vocab()
vocab.load_dict(path=os.getenv("APP_MODEL_VOCAB_PATH"))
self.log.debug('Loading CDB ...')
cdb = CDB()
cdb.load_dict(path=os.getenv("APP_MODEL_CDB_PATH"))
# Apply CUI filter if provided
if os.getenv("APP_MODEL_CUI_FILTER_PATH") is not None:
self.log.debug('Applying CDB CUI filter ...')
with open(os.getenv("APP_MODEL_CUI_FILTER_PATH")) as cui_file:
all_lines = (line.rstrip() for line in cui_file)
selected_cuis = [line for line in all_lines
if line] # filter blank lines
cdb.filter_by_cui(selected_cuis)
# Meta-annotation models are optional
meta_models = []
if os.getenv("APP_MODEL_META_PATH_LIST") is not None:
self.log.debug('Loading META annotations ...')
for model_path in os.getenv("APP_MODEL_META_PATH_LIST").split(':'):
m = MetaCAT(save_dir=model_path)
m.load()
meta_models.append(m)
return CAT(cdb=cdb, vocab=vocab, meta_cats=meta_models)
def __init__(self):
super().__init__()
self.log.info('Initializing MedCAT processor ...')
self.app_name = 'MedCAT'
self.app_lang = 'en'
self.app_version = MedCatProcessor._get_medcat_version()
self.app_model = os.getenv("APP_MODEL_NAME", 'unknown')
self.vocab = Vocab()
self.cdb = CDB()
self.cdb.load_dict(
os.getenv("APP_MODEL_CDB_PATH", '/cat/models/cdb.dat'))
self.vocab.load_dict(
path=os.getenv("APP_MODEL_VOCAB_PATH", '/cat/models/vocab.dat'))
self.cat = CAT(self.cdb, vocab=self.vocab)
self.cat.spacy_cat.train = os.getenv("APP_TRAINING_MODE", False)
self.bulk_nproc = int(os.getenv('APP_BULK_NPROC', 8))
self.log.info('MedCAT processor is ready')
def load_model(self,
model_full_tag_name,
vocab_input_file_name="vocab.dat",
cdb_input_file_name="cdb.dat"):
""" Loads variables of this object
This is used to search the site-packages models folder for installed models..
"""
vocab = Vocab.load_model(model_full_tag_name=model_full_tag_name,
input_file_name=vocab_input_file_name)
cdb = CDB.load_model(model_full_tag_name=model_full_tag_name,
input_file_name=cdb_input_file_name)
if cdb is False or vocab is False:
log.error("Exiting...")
sys.exit()
return CAT(cdb, vocab=vocab)
class MedCatProcessor(NlpProcessor):
""""
MedCAT Processor class is wrapper over MedCAT that implements annotations extractions functionality
(both single and bulk processing) that can be easily exposed for an API.
"""
def __init__(self):
super().__init__()
self.log.info('Initializing MedCAT processor ...')
self.app_name = 'MedCAT'
self.app_lang = 'en'
self.app_version = MedCatProcessor._get_medcat_version()
self.app_model = os.getenv("APP_MODEL_NAME", 'unknown')
self.vocab = Vocab()
self.cdb = CDB()
self.cdb.load_dict(
os.getenv("APP_MODEL_CDB_PATH", '/cat/models/cdb.dat'))
self.vocab.load_dict(
path=os.getenv("APP_MODEL_VOCAB_PATH", '/cat/models/vocab.dat'))
self.cat = CAT(self.cdb, vocab=self.vocab)
self.cat.spacy_cat.train = os.getenv("APP_TRAINING_MODE", False)
self.bulk_nproc = int(os.getenv('APP_BULK_NPROC', 8))
self.log.info('MedCAT processor is ready')
def get_app_info(self):
"""
Returns general information about the application
:return: application information stored as KVPs
"""
return {
'name': self.app_name,
'language': self.app_lang,
'version': self.app_version,
'model': self.app_model
}
def process_content(self, content):
"""
Processes a single document extracting the annotations.
:param content: document to be processed, containing 'text' field.
:return: processing result containing document with extracted annotations stored as KVPs.
"""
if 'text' not in content:
error_msg = "'text' field missing in the payload content."
nlp_result = {
'success': False,
'errors': [error_msg],
'timestamp': NlpProcessor._get_timestamp()
}
return nlp_result, False
text = content['text']
# assume an that a blank document is a valid document and process it only
# when it contains any non-blank characters
if text is not None and len(text.strip()) > 0:
entities = self.cat.get_entities(text)
else:
entities = []
nlp_result = {
'text': text,
'annotations': entities,
'success': True,
'timestamp': NlpProcessor._get_timestamp()
}
# append the footer
if 'footer' in content:
nlp_result['footer'] = content['footer']
return nlp_result
def process_content_bulk(self, content):
"""
Processes an array of documents extracting the annotations.
:param content: document to be processed, containing 'text' field.
:return: processing result containing documents with extracted annotations,stored as KVPs.
"""
# process at least 10 docs per thread and don't bother with starting
# additional threads when less documents were provided
min_doc_per_thread = 10
num_slices = max(1, int(len(content) / min_doc_per_thread))
batch_size = min(300, num_slices)
if batch_size >= self.bulk_nproc:
nproc = self.bulk_nproc
else:
batch_size = min_doc_per_thread
nproc = max(1, num_slices)
if len(content) > batch_size * nproc:
nproc += 1
# use generators both to provide input documents and to provide resulting annotations
# to avoid too many mem-copies
invalid_doc_ids = []
ann_res = self.cat.multi_processing(
MedCatProcessor._generate_input_doc(content, invalid_doc_ids),
nproc=nproc,
batch_size=batch_size)
return MedCatProcessor._generate_result(content, ann_res,
invalid_doc_ids)
# helper generator functions to avoid multiple copies of data
#
@staticmethod
def _generate_input_doc(documents, invalid_doc_idx):
"""
Generator function returning documents to be processed as a list of tuples:
(idx, text), (idx, text), ...
Skips empty documents and reports their ids to the invalid_doc_idx array
:param documents: array of input documents that contain 'text' field
:param invalid_doc_idx: array that will contain invalid document idx
:return: consecutive tuples of (idx, document)
"""
for i in range(0, len(documents)):
# assume the document to be processed only when it is not blank
if 'text' in documents[i] and documents[i][
'text'] is not None and len(
documents[i]['text'].strip()) > 0:
yield i, documents[i]['text']
else:
invalid_doc_idx.append(i)
@staticmethod
def _generate_result(in_documents, annotations, invalid_doc_idx):
"""
Generator function merging the resulting annotations with the input documents.
The result for documents that were invalid will not contain any annotations.
:param in_documents: array of input documents that contain 'text' field
:param annotations: array of annotations extracted from documents
:param invalid_doc_idx: array of invalid document idx
:return:
"""
# generate output for valid annotations
for i in range(len(annotations)):
res = annotations[i]
res_idx = res[0]
in_ct = in_documents[res_idx]
# parse the result
out_res = {
'text': res[1]["text"],
'annotations': res[1]["entities"],
'success': True,
'timestamp': NlpProcessor._get_timestamp()
}
# append the footer
if 'footer' in in_ct:
out_res['footer'] = in_ct['footer']
yield out_res
# generate output for invalid documents
for i in invalid_doc_idx:
in_ct = in_documents[i]
out_res = {
'text': in_ct["text"],
'annotations': [],
'success': True,
'timestamp': NlpProcessor._get_timestamp()
}
# append the footer
if 'footer' in in_ct:
out_res['footer'] = in_ct['footer']
yield out_res
@staticmethod
def _get_medcat_version():
"""
Returns the version string of the MedCAT module as reported by pip
:return:
"""
try:
import subprocess
result = subprocess.check_output(['pip', 'show', 'medcat'],
universal_newlines=True)
version_line = list(
filter(lambda v: 'Version' in v, result.split('\n')))
return version_line[0].split(' ')[1]
except Exception:
raise Exception("Cannot read the MedCAT library version")
from flask import Flask
from flask import Response
import json
from medcat.cdb import CDB
from medcat.utils.vocab import Vocab
from medcat.cat import CAT
from flask import request
import os
vocab = Vocab()
cdb = CDB()
cdb.load_dict(os.getenv("CDB_MODEL", '/cat/models/med_ann_norm.dat'))
vocab.load_dict(
path=os.getenv("VOCAB_MODEL", '/cat/models/med_ann_norm_dict.dat'))
cat = CAT(cdb, vocab=vocab)
cat.spacy_cat.train = False
app = Flask(__name__)
app_name = 'MEDCAT'
app_lang = 'en'
app_version = os.getenv("CAT_VERSION", '0.1.0')
@app.route('/api/info', methods=['GET'])
def info():
app_info = {'name': app_name, 'language': app_lang, 'version': app_version}
return Response(response=json.dumps(app_info),
status=200,
def run_cv(cdb_path,
data_path,
vocab_path,
cv=100,
nepochs=16,
reset_cui_count=True,
test_size=0.1):
from medcat.cat import CAT
from medcat.utils.vocab import Vocab
from medcat.cdb import CDB
import json
f1s = {}
ps = {}
rs = {}
tps = {}
fns = {}
fps = {}
cui_counts = {}
for i in range(cv):
cdb = CDB()
cdb.load_dict(cdb_path)
vocab = Vocab()
vocab.load_dict(path=vocab_path)
cat = CAT(cdb, vocab=vocab)
cat.train = False
cat.spacy_cat.MIN_ACC = 0.30
cat.spacy_cat.MIN_ACC_TH = 0.30
fp, fn, tp, p, r, f1, cui_counts = cat.train_supervised(
data_path=data_path,
lr=1,
nepochs=nepochs,
anneal=True,
print_stats=True,
use_filters=True,
reset_cui_count=reset_cui_count,
terminate_last=True,
test_size=test_size)
for key in f1.keys():
if key in f1s:
f1s[key].append(f1[key])
else:
f1s[key] = [f1[key]]
if key in ps:
ps[key].append(p[key])
else:
ps[key] = [p[key]]
if key in rs:
rs[key].append(r[key])
else:
rs[key] = [r[key]]
if key in tps:
tps[key].append(tp.get(key, 0))
else:
tps[key] = [tp.get(key, 0)]
if key in fps:
fps[key].append(fp.get(key, 0))
else:
fps[key] = [fp.get(key, 0)]
if key in fns:
fns[key].append(fn.get(key, 0))
else:
fns[key] = [fn.get(key, 0)]
return fps, fns, tps, ps, rs, f1s, cui_counts
continue
intersection_space = len(source.intersection(target))
target_space = len(target)
stats_dict[(mrn_number,
account_number)] = intersection_space / target_space
return stats_dict, error_log
if __name__ == '__main__':
print('Loading the vocabulary...')
try:
vocab = Vocab()
vocab.load_dict(path_medcat + '/vocab.dat')
except:
raise ImportError('vocab and script should be in same directory')
print('Loading the weights. This will take time...')
try:
cdb = CDB()
cdb.load_dict(path_medcat +
'/umls_base_wlink_fixed_x_avg_2m_mimic.dat')
except:
raise ImportError('weights and script should be in same directory')
print('Building the model...')
cat = CAT(cdb=cdb, vocab=vocab)
class MakeVocab(object):
r'''
Create a new vocab from a text file. To make a vocab and train word embeddings do:
>>>cdb = <your existing cdb>
>>>maker = MakeVocab(cdb=cdb)
>>>maker.make(data_iterator, out_folder="./output/")
>>>maker.add_vectors(self, in_path="./output/data.txt")
>>>
Args:
cdb (medcat.cdb.CDB):
The concept database that will be added ontop of the Vocab built from the text file.
vocab (medcat.utils.vocab.Vocab, optional):
Vocabulary to be extended, leave as None if you want to make a new Vocab. Default: None
word_tokenizer (<function>):
A custom tokenizer for word spliting - used if embeddings are BERT or similar.
Default: None
'''
def __init__(self, cdb=None, vocab=None, word_tokenizer=None):
self.cdb = cdb
self.w2v = None
self.vocab_path = "./vocab.dat"
if vocab is not None:
self.vocab = vocab
else:
self.vocab = Vocab()
# Build the required spacy pipeline
self.nlp = SpacyPipe(spacy_split_all, disable=['ner', 'parser', 'vectors', 'textcat'])
# Get the tokenizer
if word_tokenizer is not None:
self.tokenizer = word_tokenizer
else:
self.tokenizer = self._tok
def _tok(self, text):
return [text]
def make(self, iter_data, out_folder, join_cdb=True):
r'''
Make a vocab - without vectors initially. This will create two files in the out_folder:
- vocab.dat -> The vocabulary without vectors
- data.txt -> The tokenized dataset prepared for training of word2vec or similar embeddings.
Args:
iter_data (Iterator):
An iterator over sentences or documents. Can also be a simple array of text documents/sentences.
out_folder (string):
A path to a folder where all the results will be saved
join_cdb (bool):
Should the words from the CDB be added to the Vocab. Default: True
'''
# Save the preprocessed data, used for emb training
out_path = out_folder + "data.txt"
vocab_path = out_folder + "vocab.dat"
self.vocab_path = vocab_path
out = open(out_path, 'w')
for ind, doc in enumerate(iter_data):
if ind % 10000 == 0:
log.info("Vocab builder at: " + str(ind))
print(ind)
doc = self.nlp.nlp.tokenizer(doc)
line = ""
for token in doc:
if token.is_space or token.is_punct:
continue
if len(token.lower_) > 0:
self.vocab.inc_or_add(token.lower_)
line = line + " " + "_".join(token.lower_.split(" "))
out.write(line.strip())
out.write("\n")
out.close()
if join_cdb and self.cdb:
for word in self.cdb.vocab.keys():
if word not in self.vocab:
self.vocab.add_word(word)
else:
# Update the count with the counts from the new dataset
self.cdb.vocab[word] += self.vocab[word]
# Save the vocab also
self.vocab.save_dict(path=vocab_path)
def add_vectors(self, in_path=None, w2v=None, overwrite=False, data_iter=None, workers=8, niter=2, min_count=10, window=10, vsize=300):
r'''
Add vectors to an existing vocabulary and save changes to the vocab_path.
Args:
in_path (String):
Path to the data.txt that was created by the MakeVocab.make() function.
w2v (Word2Vec, optional):
An existing word2vec instance. Default: None
overwrite (bool):
If True it will overwrite existing vectors in the vocabulary. Default: False
data_iter (iterator):
If you want to provide a customer iterator over the data use this. If yes, then in_path is not needed.
**: Word2Vec arguments
Returns:
A trained word2vec model.
'''
if w2v is None:
if data_iter is None:
data = SimpleIter(in_path)
else:
data = data_iter
w2v = Word2Vec(data, window=window, min_count=min_count, workers=workers, size=vsize, iter=niter)
for word in w2v.wv.vocab.keys():
if word in self.vocab:
if overwrite:
self.vocab.add_vec(word, w2v.wv.get_vector(word))
else:
if self.vocab.vec(word) is None:
self.vocab.add_vec(word, w2v.wv.get_vector(word))
# Save the vocab again, now with vectors
self.vocab.make_unigram_table()
self.vocab.save_dict(path=self.vocab_path)
return w2v
class MakeVocab(object):
SKIP_STOPWORDS = False
def __init__(self, cdb, vocab=None, word_tokenizer=None):
self.cdb = cdb
self.w2v = None
if vocab is not None:
self.vocab = vocab
else:
self.vocab = Vocab()
# Build the required spacy pipeline
self.nlp = SpacyPipe(spacy_split_all, disable=['ner', 'parser', 'vectors', 'textcat'])
self.nlp.add_punct_tagger(tagger=partial(spacy_tag_punct, skip_stopwords=self.SKIP_STOPWORDS))
self.spell_checker = CustomSpellChecker(cdb_vocab=cdb.vocab, data_vocab=vocab)
self.nlp.add_spell_checker(spell_checker=self.spell_checker)
# Get the tokenizer
if word_tokenizer is not None:
self.tokenizer = word_tokenizer
else:
self.tokenizer = self._tok
def _tok(self, text):
return [text]
def make(self, iter_data, out_folder, join_cdb=True):
# Save the preprocessed data, used for emb training
out_path = out_folder + "data.txt"
vocab_path = out_folder + "vocab.dat"
out = open(out_path, 'w')
for ind, doc in enumerate(iter_data):
if ind % 10000 == 0:
print(ind)
doc = self.nlp(doc)
line = ""
for token in doc:
if token._.to_skip:
continue
if len(token._.norm) > 1:
self.vocab.inc_or_add(token._.norm)
# Add also the unnormalized version if it is different
if token._.norm != token.lower_:
self.vocab.inc_or_add(token.lower_)
line = line + " " + "_".join(token._.norm.split(" "))
out.write(line.strip())
out.write("\n")
out.close()
if join_cdb and self.cdb:
for word in self.cdb.vocab.keys():
if word not in self.vocab:
self.vocab.add_word(word)
else:
# Update the count with the counts from the new dataset
self.cdb.vocab[word] += self.vocab[word]
# Save the vocab also
self.vocab.save_dict(path=vocab_path)
def add_vectors(self, in_path, overwrite=False):
data = SimpleIter(in_path)
w2v = Word2Vec(data, window=10, min_count=10, workers=8, size=300, iter=2)
for word in w2v.wv.vocab.keys():
if word in self.vocab:
if overwrite:
self.vocab.add_vec(word, w2v.wv.get_vector(word))
else:
if self.vocab.vec(word) is None:
self.vocab.add_vec(word, w2v.wv.get_vector(word))
return w2v
mini_str = '_small' if args.mini else ''
snomed_core_fn = '../data/SNOMEDCT_CORE_SUBSET_202005/SNOMEDCT_CORE_SUBSET_202005.txt'
semgroups_fn = '../data/umls_semgroups.txt'
cols = ['UMLS_CUI', 'SNOMED_FSN', 'SNOMED_CID']
snomed_df = pd.read_csv(snomed_core_fn, delimiter='|')[cols]
core_cui_set = set(snomed_df['UMLS_CUI'].tolist())
# sem_group_acronym|sem_group|tui|tui_description
sem_groups_df = pd.read_csv(semgroups_fn, delimiter='|').dropna()
tui_group_map = dict(
zip(sem_groups_df['tui'].tolist(),
sem_groups_df['sem_group'].tolist()))
vocab = Vocab()
print('Loading vocabulary...')
# Load the vocab model you downloaded
vocab.load_dict('../data/medcat/vocab.dat')
# Load the cdb model you downloaded
cdb = CDB()
print('Loading model...')
cdb.load_dict('../data/medcat/cdb.dat')
# create cat
print('Creating MedCAT pipeline...')
cat = CAT(cdb=cdb, vocab=vocab)
print('Loading Spacy...')
sentencizer = spacy.load(
def run_cv(cdb_path, data_path, vocab_path, cv=100, nepochs=16, test_size=0.1, lr=1, groups=None, **kwargs):
from medcat.cat import CAT
from medcat.utils.vocab import Vocab
from medcat.cdb import CDB
import json
use_groups = False
if groups is not None:
use_groups = True
f1s = {}
ps = {}
rs = {}
tps = {}
fns = {}
fps = {}
cui_counts = {}
examples = {}
for i in range(cv):
cdb = CDB()
cdb.load_dict(cdb_path)
vocab = Vocab()
vocab.load_dict(path=vocab_path)
cat = CAT(cdb, vocab=vocab)
cat.train = False
cat.spacy_cat.MIN_ACC = 0.30
cat.spacy_cat.MIN_ACC_TH = 0.30
# Add groups if they exist
if groups is not None:
for cui in cdb.cui2info.keys():
if "group" in cdb.cui2info[cui]:
del cdb.cui2info[cui]['group']
groups = json.load(open("./groups.json"))
for k,v in groups.items():
for val in v:
cat.add_cui_to_group(val, k)
fp, fn, tp, p, r, f1, cui_counts, examples = cat.train_supervised(data_path=data_path,
lr=1, test_size=test_size, use_groups=use_groups, nepochs=nepochs, **kwargs)
for key in f1.keys():
if key in f1s:
f1s[key].append(f1[key])
else:
f1s[key] = [f1[key]]
if key in ps:
ps[key].append(p[key])
else:
ps[key] = [p[key]]
if key in rs:
rs[key].append(r[key])
else:
rs[key] = [r[key]]
if key in tps:
tps[key].append(tp.get(key, 0))
else:
tps[key] = [tp.get(key, 0)]
if key in fps:
fps[key].append(fp.get(key, 0))
else:
fps[key] = [fp.get(key, 0)]
if key in fns:
fns[key].append(fn.get(key, 0))
else:
fns[key] = [fn.get(key, 0)]
return fps, fns, tps, ps, rs, f1s, cui_counts, examples
def load_model_from_file(full_model_tag_name="",
file_name="",
model_folder=".",
bypass_model_path=False):
"""
Looks into the models directory in your /site-packages/medcat-{version}/model_name/ installation.
:param full_model_tag_name: self-explanatory
:param file_name: the model file name that we want to load e.g: "vocab.dat", "cdb.dat", "MedCAT_export.json" etc
:param model_folder: path to model folder, the default is medcat's package folder,
use bypass_model_path=True if you wish to specify your own path
:param bypass_model_path: will look into specified folder instead of model folder
:return: file data
"""
local_model_folder_path = False
if full_model_tag_name != "":
local_model_folder_path = get_downloaded_local_model_folder(
full_model_tag_name)
full_file_path = False
if local_model_folder_path:
full_file_path = os.path.join(local_model_folder_path, file_name)
if bypass_model_path is True:
full_file_path = os.path.join(model_folder, file_name)
data = False
if full_file_path:
with open(full_file_path, "rb") as f:
data = pickle.load(f)
version = ""
model_name = ""
if full_model_tag_name != "":
model_name, version = get_tag_str_model_name_and_version(
full_model_tag_name)
try:
obj = None
fname = str(file_name).lower()
if isinstance(data, dict):
from medcat.cdb import CDB
from medcat.utils.vocab import Vocab
if "vocab" in fname:
obj = Vocab()
if "cdb" in fname:
obj = CDB()
obj.__dict__ = data
if obj is not None:
if not hasattr(obj, "vc_model_tag_data"):
obj.vc_model_tag_data = ""
obj.vc_model_tag_data = ModelTagData(
model_name=model_name, version=version)
data = obj
except Exception as exception:
logging.error(
"could not add vc_model_tag_data attribute to model data file"
)
logging.error(repr(exception))
return False
return data
class MakeVocab(object):
def __init__(self, cdb, vocab=None, word_tokenizer=None):
self.cdb = cdb
self.w2v = None
if vocab is not None:
self.vocab = vocab
else:
self.vocab = Vocab()
# Build the required spacy pipeline
self.nlp = SpacyPipe(spacy_split_all,
disable=['ner', 'parser', 'vectors', 'textcat'])
# Get the tokenizer
if word_tokenizer is not None:
self.tokenizer = word_tokenizer
else:
self.tokenizer = self._tok
def _tok(self, text):
return [text]
def make(self, iter_data, out_folder, join_cdb=True):
# Save the preprocessed data, used for emb training
out_path = out_folder + "data.txt"
vocab_path = out_folder + "vocab.dat"
out = open(out_path, 'w')
for ind, doc in enumerate(iter_data):
if ind % 10000 == 0:
log.info("Vocab builder at: " + str(ind))
print(ind)
doc = self.nlp.nlp.tokenizer(doc)
line = ""
for token in doc:
if token.is_space or token.is_punct:
continue
if len(token.lower_) > 0:
self.vocab.inc_or_add(token.lower_)
line = line + " " + "_".join(token.lower_.split(" "))
out.write(line.strip())
out.write("\n")
out.close()
if join_cdb and self.cdb:
for word in self.cdb.vocab.keys():
if word not in self.vocab:
self.vocab.add_word(word)
else:
# Update the count with the counts from the new dataset
self.cdb.vocab[word] += self.vocab[word]
# Save the vocab also
self.vocab.save_dict(path=vocab_path)
def add_vectors(self,
in_path=None,
w2v=None,
overwrite=False,
workers=8,
niter=2,
min_count=10,
window=10,
vsize=300):
if w2v is None:
data = SimpleIter(in_path)
w2v = Word2Vec(data,
window=window,
min_count=min_count,
workers=workers,
size=vsize,
iter=niter)
for word in w2v.wv.vocab.keys():
if word in self.vocab:
if overwrite:
self.vocab.add_vec(word, w2v.wv.get_vector(word))
else:
if self.vocab.vec(word) is None:
self.vocab.add_vec(word, w2v.wv.get_vector(word))
return w2v
# TODO
#neg_path = os.getenv('NEG_PATH', '/tmp/mc_negated')
try:
if not os.path.exists(vocab_path):
vocab_url = os.getenv('VOCAB_URL')
urlretrieve(vocab_url, vocab_path)
if not os.path.exists(cdb_path):
cdb_url = os.getenv('CDB_URL')
print("*" * 399)
print(cdb_url)
urlretrieve(cdb_url, cdb_path)
vocab = Vocab()
vocab.load_dict(vocab_path)
cdb = CDB()
cdb.load_dict(cdb_path)
# mc_negated = MetaCAT(save_dir=neg_path)
# mc_negated.load()
# cat = CAT(cdb=cdb, vocab=vocab, meta_cats=[mc_negated])
cat = CAT(cdb=cdb, vocab=vocab)
cat.spacy_cat.MIN_ACC = 0.30
cat.spacy_cat.MIN_ACC_TH = 0.30
cat.spacy_cat.ACC_ALWAYS = True
except Exception as e:
print(str(e))
def get_html_and_json(text):
import os
from argparse import ArgumentParser
import pandas as pd
from tqdm import tqdm
import numpy as np
from medcat.cat import CAT
from medcat.utils.vocab import Vocab
from medcat.cdb import CDB
vocab = Vocab()
vocab.load_dict(os.environ["MEDCAT_VOCAB_FILE"])
print("Loaded Vocab")
# Load the cdb model you downloaded
cdb = CDB()
cdb.load_dict(os.environ["MEDCAT_CDB_FILE"])
print("Loaded CDB")
# create cat
cat = CAT(cdb=cdb, vocab=vocab)
cat.spacy_cat.TUI_FILTER = ['T047', 'T048', 'T184']
tqdm.pandas()
def get_entities(text) :
doc = cat.get_entities(text)
relevant_entities = []
for ent in doc :
if "icd10" in ent["info"] :
from code_utils.global_variables import * from medcat.cat import CAT from medcat.utils.vocab import Vocab from medcat.prepare_cdb import PrepareCDB from medcat.cdb import CDB import os import spacy # nlp = spacy.load(spacy_en_path, disable=['ner', 'parser']) medcat_path = r'C:\Users\K1774755\PycharmProjects\toy-models\MedCat' vocab = Vocab() # Load the vocab model you just downloaded vocab.load_dict(os.path.join(medcat_path, 'med_ann_norm_dict.dat')) # If you have an existing CDB cdb = CDB() # cdb.load_dict(os.path.join(medcat_path, 'simple_cdb.csv')) # If you need a special CDB you can build one from a .csv file preparator = PrepareCDB(vocab=vocab) csv_paths = [os.path.join(medcat_path, 'simple_cdb.csv')]#, '<another one>', ...] csv_paths = [os.path.join(medcat_path, 'attention_cdb.csv')] cdb = preparator.prepare_csvs(csv_paths) # Save the new CDB for later cdb.save_dict(os.path.join(medcat_path, 'simple_cdb.cdb')) # To annotate documents we do doc = "My simple document with kidney failure"