def output_similar_strings_of_each_line(path, measure):
strings = []
with open(path, "r") as lines:
for line in lines:
strings.append(line.rstrip("\r\n"))
db = DictDatabase(CharacterNgramFeatureExtractor(2))
for string in strings:
db.add(string)
db.save("companies.db")
dbl = DictDatabase.load("companies.db")
searcher = Searcher(dbl, measure)
profiler.start()
for string in strings:
result = searcher.search(string, 0.8)
# result = [str(np.round(x[0], 5)) + ' ' + x[1] for x in searcher.ranked_search(string, 0.8)]
# print("\t".join([string, ",".join(result)]))
profiler.stop()
profiler.print()
profiler.open_in_browser()
class TestDict(TestCase):
strings = ['a', 'ab', 'abc', 'abcd', 'abcde']
def setUp(self):
self.db = DictDatabase(CharacterNgramFeatureExtractor(2))
for string in self.strings:
self.db.add(string)
def test_strings(self):
self.assertEqual(self.db.strings, self.strings)
def test_min_feature_size(self):
self.assertEqual(self.db.min_feature_size(),
min(map(lambda x: len(x) + 1, self.strings)))
def test_max_feature_size(self):
self.assertEqual(self.db.max_feature_size(),
max(map(lambda x: len(x) + 1, self.strings)))
def test_lookup_strings_by_feature_set_size_and_feature(self):
self.assertEqual(
self.db.lookup_strings_by_feature_set_size_and_feature(4, 'ab_1'),
set(['abc']))
self.assertEqual(
self.db.lookup_strings_by_feature_set_size_and_feature(3, 'ab_1'),
set(['ab']))
self.assertEqual(
self.db.lookup_strings_by_feature_set_size_and_feature(2, 'ab_1'),
set([]))
def get_umls_data():
umls_data = pd.read_csv(umls_df_data_path)
print(f"Got UMLS data at length {len(umls_data)}")
acronyms_umls_df = pd.read_csv(acronyms_dir + os.sep + 'acronyms_terms.csv')
umls_data = pd.concat([umls_data, acronyms_umls_df])
cuiless_umls_df = pd.read_csv(cuiless_dir + os.sep + 'cuiless_terms.csv')
umls_data = pd.concat([umls_data, cuiless_umls_df])
umls_data.reset_index(inplace=True)
heb_umls_list = list(umls_data['HEB'].values)
eng_umls_list = list(umls_data[STRING_COLUMN].values)
heb_db = DictDatabase(CharacterNgramFeatureExtractor(2))
eng_db = DictDatabase(CharacterNgramFeatureExtractor(2))
for heb_w in heb_umls_list:
heb_db.add(heb_w)
for eng_w in eng_umls_list:
lower_eng_w = eng_w.lower()
eng_db.add(lower_eng_w)
return heb_db, eng_db, umls_data
def create_umls_ss_db(umls_kb, char_ngram_len=3, n_max_tokens=5):
logging.info('Loading scispacy ...')
import spacy
self.sci_nlp = spacy.load('en_core_sci_md',
disable=['tagger', 'parser', 'ner'])
simstring_db = DictDatabase(CharacterNgramFeatureExtractor(char_ngram_len))
# preprocessing aliases and labels
logging.info('Preprocessing aliases ...')
alias_mapping = defaultdict(set)
aliases = []
for cui in umls_kb.get_all_cuis():
cui_aliases = set(
[a.lower() for a in umls_kb.get_aliases(cui, include_name=True)])
for alias in cui_aliases:
alias_chars = set(alias)
if len(alias_chars.intersection(fb_punctuation)) > 0:
continue
elif alias in en_stopwords:
continue
elif alias.isnumeric():
continue
alias_doc = self.sci_nlp(
alias) # use same tokenizer as when splitting medmentions
if len(alias_doc
) > n_max_tokens: # gets too big without restrictions
continue
alias_mapping[alias].add(cui)
aliases.append(alias)
logging.info('Adding to DB ...')
for alias_idx, alias in enumerate(aliases):
simstring_db.add(alias)
if alias_idx % 1000000 == 0:
logging.info('At %d/%d ...' % (alias_idx, len(aliases)))
# setting paths
db_path = '%s.aliases.%dgram.%dtoks.db' % (umls_kb.umls_version,
char_ngram_len, n_max_tokens)
map_path = '%s.aliases.%dtoks.map' % (umls_kb.umls_version, n_max_tokens)
logging.info('Storing DB ...')
with open(db_path, 'wb') as f:
pickle.dump(simstring_db, f)
logging.info('Storing Alias Mapping ...')
with open(map_path, 'wb') as f:
alias_mapping = dict(alias_mapping)
pickle.dump(alias_mapping, f)
def load_disambiguation():
db = DictDatabase(WordNgramFeatureExtractor(2))
with open(LOCATION_WIKIPEDIA_DISAMBIGUATION) as disambig_file:
for line in disambig_file:
r = line.replace("_(disambiguation)", "").replace("_", " ").lower()
db.add(r.strip())
return Searcher(db, JaccardMeasure())
class MagnitudeOOV():
def __init__(self, word2vec):
self.w2v = word2vec
self.embedding_dim = self.w2v.vector_size
self.vocab = set(self.w2v.vocab.keys())
self.db = DictDatabase(CharacterNgramFeatureExtractor(2))
for vocab_word in self.vocab:
self.db.add(vocab_word)
def generate_pseudorandom_vector(self, word):
"""calculate PRVG form CGRAM"""
vectors = []
ngram_list = character_ngram(word)
for ngram in ngram_list:
np.random.seed(seed(ngram))
vectors.append(np.random.uniform(-1, 1, self.embedding_dim))
return np.mean(vectors, axis=0)
def similar_words_top_k(self, query, measure=CosineMeasure(), initial_threshold=0.99, dec_step=0.01, k=3):
"""search similar words by using edit distance"""
searcher = Searcher(self.db, measure)
t = initial_threshold
similar_words = []
while True:
similar_words = searcher.search(query, t)
if len(similar_words) >= k or t <= 0.1:
break
t -= dec_step
if len(similar_words) > 3:
np.random.choice(42)
return np.random.choice(similar_words, k, replace=False).tolist()
else:
return similar_words
def generate_similar_words_vector(self, word):
"""calculate MATCH from similar words"""
vectors = np.mean([self.w2v[w] for w in self.similar_words_top_k(word)], axis=0)
return vectors
def out_of_vocab_vector(self, word):
vector = self.generate_pseudorandom_vector(word) * 0.3 + self.generate_similar_words_vector(word) * 0.7
final_vector = vector / np.linalg.norm(vector)
return final_vector
def query(self, word):
normalized_word = neologdn.normalize(word)
if word in self.vocab:
return self.w2v[word]
elif normalized_word in self.vocab:
return self.w2v[normalized_word]
else:
return self.out_of_vocab_vector(normalized_word)
def output_similar_strings_of_each_line(path):
db = DictDatabase(CharacterNgramFeatureExtractor(2))
with open(path, 'r') as lines:
for line in lines:
strings = line.rstrip('\r\n')
db.add(strings)
searcher = Searcher(db, CosineMeasure())
with open(path, 'r') as lines:
for line in lines:
strings = line.rstrip('\r\n')
result = [str(round(x[0], 5)) + ' ' + x[1] for x in searcher.ranked_search(strings, 0.8)]
print("\t".join([strings, ",".join(result)]))
def simstring_database(umls, nchar_val):
db = DictDatabase(CharacterNgramFeatureExtractor(nchar_val))
term_to_cui = dict()
for value in umls:
try:
data = value.split('\t')
cui = data[0]
term = data[1].lower()
db.add(term)
term_to_cui[term] = cui
except:
continue
pickle.dump(db, open('db.pickle', 'wb'))
pickle.dump(term_to_cui, open('term_to_cui.pickle', 'wb'))
def output_similar_strings_of_each_line(path):
db = DictDatabase(CharacterNgramFeatureExtractor(2))
with open(path, 'r') as lines:
for line in lines:
strings = line.rstrip('\r\n')
db.add(strings)
searcher = Searcher(db, CosineMeasure())
with open(path, 'r') as lines:
for line in lines:
strings = line.rstrip('\r\n')
result = [
str(round(x[0], 5)) + ' ' + x[1]
for x in searcher.ranked_search(strings, 0.8)
]
print("\t".join([strings, ",".join(result)]))
def setUp(self) -> None:
db = DictDatabase(CharacterNgramFeatureExtractor(2))
db.add("foo")
db.add("bar")
db.add("fooo")
db.add("food")
db.add("fool")
db.add("follow")
self.searcher = Searcher(db, JaccardMeasure())
def construct_ontology(ontology_data):
'''
Create an n-char simstring database and
term-to-code mapping to enable rapid ontology
querying
'''
database = DictDatabase(CharacterNgramFeatureExtractor(2))
term_to_cui = {}
for entry in ontology_data:
entry_values = entry.split('\t')
if len(entry_values) == 2:
term = clean_selected_term(entry_values[1])
term_to_cui[term] = entry_values[0].strip()
for term in term_to_cui.keys():
term = clean_selected_term(term)
database.add(term)
return database, term_to_cui
class TestDict(TestCase):
strings = ['a', 'ab', 'abc', 'abcd', 'abcde']
def setUp(self):
self.db = DictDatabase(CharacterNgramFeatureExtractor(2))
for string in self.strings:
self.db.add(string)
def test_strings(self):
self.assertEqual(self.db.strings, self.strings)
def test_min_feature_size(self):
self.assertEqual(self.db.min_feature_size(), min(map(lambda x: len(x) + 1, self.strings)))
def test_max_feature_size(self):
self.assertEqual(self.db.max_feature_size(), max(map(lambda x: len(x) + 1, self.strings)))
def test_lookup_strings_by_feature_set_size_and_feature(self):
self.assertEqual(self.db.lookup_strings_by_feature_set_size_and_feature(4, 'ab'), set(['abc']))
self.assertEqual(self.db.lookup_strings_by_feature_set_size_and_feature(3, 'ab'), set(['ab']))
self.assertEqual(self.db.lookup_strings_by_feature_set_size_and_feature(2, 'ab'), set([]))
def make_change_image_dict(drink_names):
import re
import json
import difflib
from simstring.feature_extractor.character_ngram import CharacterNgramFeatureExtractor
from simstring.measure.cosine import CosineMeasure
from simstring.database.dict import DictDatabase
from simstring.searcher import Searcher
ff = open('jsons/theCocktailDB_allData_20181010.json', 'r', encoding="utf-8_sig")
json_data2 = json.load(ff)
ff.close()
# 互いに類似度を比較する文字列のリスト
STR_db = [re.sub(r'[!-/:-@[-`{-~]', " ", d["en"]) for d in drink_names]
TCD_db ={re.sub(r'[!-/:-@[-`{-~]', " ", d["drinks"][0]["strDrink"]): d["drinks"][0]["strDrinkThumb"] for d in json_data2}
TCD_name_db = list(TCD_db.keys())
count = 0
length = len(STR_db)
result_dict = {}
change_image_dict = {}
db = DictDatabase(CharacterNgramFeatureExtractor(2))
for str1 in STR_db:
db.add(str1)
for str2 in TCD_name_db:
result_dict[str2] = {}
searcher = Searcher(db, CosineMeasure())
i = 1.0
# 類似度を計算、0.0~1.0 で結果が返る
flag = False
for str1 in STR_db:
s = difflib.SequenceMatcher(None, str2, str1).ratio()
if s > 0.75:
flag = True
if (str1 in result_dict[str2]):
d = result_dict[str2][str1]
#平均更新
d = [(d[0]*d[1]+s)/(d[1]+1), d[1]+1]
result_dict[str2][str1] = d
else:
result_dict[str2].setdefault(str1, [s ,1])
temp = []
while i >= 0.65:
result = searcher.search(str2, i)
if (len(result)):
flag = True
for str1 in result:
if (str1 in temp): continue
temp += [str1]
if (str1 in result_dict[str2]):
d = result_dict[str2][str1]
#平均更新
d = [(d[0]*d[1]+i)/(d[1]+1), d[1]+1]
result_dict[str2][str1] = d
else:
result_dict[str2].setdefault(str1, [i ,1])
i -= 0.001
if (flag):
count += 1
with open("./search_log.txt", "w+", encoding="utf-8_sig") as f:
real_count = 0
for str2 in TCD_name_db:
print("\n", file=f)
print("\n")
print(">> "+str2, file=f)
print(">> "+str2)
M = 0.0
name = ""
for key, value_list in result_dict[str2].items():
if (M < value_list[0]):
name = key
M = value_list[0]
print(" "+name+": "+str(M), file=f)
if (M != 0):
if (M >= 0.76):
print(" "+name+": "+str(M))
print("ok", file=f)
print("ok")
change_image_dict[name] = TCD_db[str2]
real_count += 1
else:
print(" "+name+": "+str(M))
print("out", file=f)
print("out")
print("\nmatch is {count}/{length} but real_match is {real_count}/{length}".format(count=count, real_count=real_count, length=length), file=f)
print("\nmatch is {count}/{length} but real_match is {real_count}/{length}".format(count=count, real_count=real_count, length=length))
exit()
return change_image_dict
def setUp(self):
db = DictDatabase(CharacterNgramFeatureExtractor(2))
for string in self.strings:
db.add(string)
self.searcher = Searcher(db, CosineMeasure())
def setUp(self):
db = DictDatabase(CharacterNgramFeatureExtractor(2))
for string in self.strings:
db.add(string)
self.searcher = Searcher(db, CosineMeasure())
class UMLSMapper:
# https://www.ncbi.nlm.nih.gov/books/NBK9685/table/ch03.T.concept_names_and_sources_file_mr/
def __init__(self,
from_dir: str = None,
json_path: str = "mapper.json",
umls_words: Iterable[str] = None):
# self.db = DictDatabase(WordNgramFeatureExtractor(2))
self.db = DictDatabase(CharacterNgramFeatureExtractor(2))
if from_dir:
json_path = os.path.join(from_dir, json_path)
if os.path.exists(json_path):
print(f"initialize {self.__class__.__name__}... Load json")
self.umls_dict, self.umls_reverse_dict = self.load_from_json(
json_path)
self.add_words_to_db(self.umls_dict.keys())
else:
print(f"initialize {self.__class__.__name__}... Load dir")
self.umls_dict, self.umls_reverse_dict = self.load_umls_dict(
from_dir)
self.add_words_to_db(self.umls_dict.keys())
self.save_as_json(path=json_path)
else:
self.add_words_to_db(umls_words)
# if from_dir:
# print(f"initialize {self.__class__.__name__}... Load dir")
# self.umls_dict, self.umls_reverse_dict = self.load_umls_dict(from_dir)
# self.add_words_to_db(self.umls_dict.keys())
# elif json_path:
# print(f"initialize {self.__class__.__name__}... Load json")
# self.umls_dict, self.umls_reverse_dict = self.load_from_json(json_path)
# self.add_words_to_db(self.umls_dict.keys())
# else:
# self.add_words_to_db(umls_words)
def load_umls_dict(self, directory):
path = os.path.join(directory, "GER_MRCONSO.RRF")
df = pd.read_csv(path, delimiter="|", header=None)
df.columns = [
"CUI", "LAT", "TS", "LUI", "STT", "SUI", "ISPREF", "AUI", "SAUI",
"SCUI", "SDUI", "SAB", "TTY", "CODE", "STR", "SRL", "SUPPRESS",
"CVF", "NONE"
]
df = df.drop(columns=['NONE'])
dic = {row["STR"]: row["CUI"] for i, row in df.iterrows()}
rev_dic = defaultdict(list)
for key, value in dic.items():
rev_dic[value].append(key)
return dic, rev_dic
def save_as_json(self, path: str):
data = self.__dict__
data.pop("db")
with open(path, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=0)
def load_from_json(self, path: str):
with open(path, 'r', encoding='utf-8') as file:
data = json.loads(file.read())
return data["umls_dict"], data["umls_reverse_dict"]
def add_words_to_db(self, words: Iterable[str]):
for token in set(words):
self.db.add(token)
def search_term_sims(self, term: str) -> List[str]:
searcher = Searcher(self.db, CosineMeasure())
return searcher.search(term, 0.8)
def search_all_term_sims(self, terms: List[str]) -> Dict[str, List[str]]:
dic = {}
for term in set(terms):
related_terms = self.search_term_sims(term)
if len(related_terms) > 0:
dic[term] = related_terms
return dic
# return {term: self.search_term_sims(term) for term in set(terms)}
def standardize_words(self, tokens: List[str]):
concept_dict = self.search_all_term_sims(tokens)
standardized_tokens = []
for token in tokens:
mapping = concept_dict.get(token)
if mapping:
standardized_tokens.append(mapping[0])
else:
standardized_tokens.append(token)
return standardized_tokens
def standardize_documents(self, documents: List[List[str]]):
concept_dict = self.search_all_term_sims(list(chain(*documents)))
standardized_documents = []
for document in tqdm(documents):
standardized_tokens = []
for token in document:
mapping = concept_dict.get(token)
if mapping:
standardized_tokens.append(mapping[0])
else:
standardized_tokens.append(token)
standardized_documents.append(standardized_tokens)
return standardized_documents
def get_umls_vectors_only(self, vectors: gensim.models.KeyedVectors):
medical_concepts = [
word for word in vectors.index2word
if word in self.umls_dict.values()
]
concept_vecs = {
concept: vectors.get_vector(concept)
for concept in medical_concepts
}
return concept_vecs
def un_umls(self, concept, single_return=True):
res = self.umls_reverse_dict.get(concept)
if res is None:
return concept
if single_return:
return res[0]
else:
return res
def replace_umls(self, tokens: List[str]) -> List[str]:
return [self.un_umls(token) for token in tokens if self.un_umls(token)]
def umls_code(self, token, delete_non_umls):
umls_code = self.umls_dict.get(token)
if umls_code is None:
if delete_non_umls:
return None
else:
return token
else:
return umls_code
def replace_with_umls(self,
tokens: List[str],
delete_non_umls=False) -> List[str]:
return [
self.umls_code(token, delete_non_umls) for token in tokens
if self.umls_code(token, delete_non_umls)
]
def replace_documents_token_based(self, documents: List[str], delete_non_umls=False, tokenize: bool = True) \
-> Union[List[List[str]], List[str]]:
tokenized_documents = [sentence.split() for sentence in documents]
if tokenize:
return [[
self.umls_code(token, delete_non_umls) for token in tokens
if self.umls_code(token, delete_non_umls)
] for tokens in tokenized_documents]
else:
return [
' '.join([
self.umls_code(token, delete_non_umls) for token in tokens
if self.umls_code(token, delete_non_umls)
]) for tokens in tokenized_documents
]
def spacy_tokenize(self,
documents: List["str"],
nlp=None) -> List[List[str]]:
if nlp is None:
nlp = spacy.load('de_core_news_sm')
doc_pipe = list(
nlp.pipe(documents, disable=["tagger", "parser", "ner"]))
tokenized_docs = [[
token.text for token in doc
] for doc in tqdm(doc_pipe, desc="Tokenize", total=len(documents))]
return tokenized_docs
def replace_documents_with_spacy_multiterm(
self,
documents: List[str],
tokenize: bool = True) -> List[List[str]]:
nlp = spacy.load('de_core_news_sm')
matcher = PhraseMatcher(nlp.vocab)
terms = self.umls_dict.keys()
doc_pipe = list(
nlp.pipe(documents, disable=["tagger", "parser", "ner"]))
# Only run nlp.make_doc to speed things up
patterns = [nlp.make_doc(term) for term in terms]
matcher.add("TerminologyList", None, *patterns)
replaced_docs = []
for doc in tqdm(doc_pipe,
desc="Replace with concepts",
total=len(documents)):
text_doc = doc.text
matches = matcher(doc)
concepts = []
for match_id, start, end in matches:
span = doc[start:end]
concepts.append(span.text)
concepts.sort(key=lambda s: len(s), reverse=True)
for concept in concepts:
text_doc = text_doc.replace(concept, self.umls_dict[concept])
replaced_docs.append(text_doc)
# tokens = [token for token in text_doc.split()]
# replaced_docs.append(tokens)
if tokenize:
replaced_docs = self.spacy_tokenize(replaced_docs, nlp)
# doc_pipe = list(nlp.pipe(replaced_docs, disable=["tagger", "parser", "ner"]))
# replaced_docs = [[token.text for token in doc] for doc in tqdm(doc_pipe, desc="Tokenize", total=len(documents))]
return replaced_docs
class GESSimpleMatcher:
'''
Clase para hacer match simple de patologías GES. Solo considera similitud entre strings,
nada muy sofisticado. Basado en código de Fabián Villena (https://fabianvillena.cl).
Actualmente considera un extractor de features que combina caracteres y palabras y tiene
ciertas cosas específicas de textos GES.
TODO:
- probar técnicas un poco más sofisticadas de matching
- completar la documentación
'''
def __init__(
self,
base_ges_data='ges_utils/data/ges-health-problems.json',
no_ges_str='UNK',
alpha=0.2,
n_chars=4,
n_words=[2],
special_words=['vih']
):
self.alpha = alpha
with open(base_ges_data,'r',encoding='utf-8') as f:
self.__ges_dict = json.load(f)
# feature extractor
extractor = GESSyntacticFeatureExtractor(
n_chars=n_chars,
n_words=n_words,
special_words=special_words
)
self.__db = DictDatabase(extractor)
# Caché
self.__cache = {}
self.__problems_from_disease = defaultdict(set)
self.__ids_from_disease = defaultdict(set)
self.__problems = {}
self.__ids = {}
self.__problems[-1] = no_ges_str
self.__ids[no_ges_str] = -1
# Por ahora los ids son el orden de los problemas en el json
# TODO: decidir si los ids deberían obtenerse de algún lugar estándar
for i, problem in enumerate(self.__ges_dict):
problem_id = i+1
self.__problems[problem_id] = problem
self.__ids[problem] = problem_id
# agrega un problema como si fuera disease también
self.__problems_from_disease[problem].add(problem)
self.__ids_from_disease[problem].add(problem_id)
# agrega a las BD
self.__db.add(problem)
for disease in self.__ges_dict[problem]:
self.__problems_from_disease[disease].add(problem)
self.__ids_from_disease[disease].add(problem_id)
# agrega a la BD
self.__db.add(disease)
# TODO: agregar datos adicionales para hacer matching de enfermedades y problemas
self.__searcher = Searcher(self.__db, CosineMeasure())
def get_ranking_ges_diseases(self, raw_string):
ranking = self.__searcher.ranked_search(raw_string, alpha=self.alpha)
return ranking
def get_ges_problem(self, raw_string):
problem_id = self.get_ges_id(raw_string)
problem = self.__problems[problem_id]
return problem
def get_ges_id(self, raw_string):
# si ya lo computamos entrega el valor
if raw_string in self.__cache:
return self.__cache[raw_string]
# si no lo tenemos, lo computamos
ranking = self.get_ranking_ges_diseases(raw_string)
if ranking:
# ipdb.set_trace()
(v, disease) = ranking[0]
problem_ids = self.__ids_from_disease[disease]
problem_id = list(problem_ids)[0]
self.__cache[raw_string] = problem_id
return problem_id
else:
self.__cache[raw_string] = -1
return -1
def get_possible_ges_ids(self, raw_string):
to_search = raw_string
problem_ids = []
# busca las enfermedades candidatas
candidate_diseases = self.__searcher.search(to_search, alpha=self.alpha)
for disease in candidate_diseases:
problem_ids.extend(self.__ids_from_disease[disease])
problem_ids_counter = Counter(problem_ids)
ordered_ids = [i for i,_ in problem_ids_counter.most_common()]
return ordered_ids
def get_ges_id_prev(self, raw_string):
# si ya lo computamos entrega el valor
if hash(raw_string) in self.__cache:
return self.__cache[hash(raw_string)]
ids_list = self.get_possible_ges_ids(raw_string)
if not ids_list:
self.__cache[raw_string] = -1
return -1
else:
self.__cache[raw_string] = ids_list[0]
return ids_list[0]
def problem_from_id(self, id_problem):
return self.__problems[id_problem]
def id_from_problem(self, problem):
return self.__ids[problem]
def clean_cache(self):
self.__cache = {}
#############################
# Read in branded foods CSV and clean it
df = pd.read_csv('branded_food.csv')
all_ingredients_final = get_cleaned_ingredients_list(df)
# Get a count for all the ingredients to be used by Peter Norvig Implementation
ingredients_count = Counter(all_ingredients_final)
##############################################
# Peter Norvig SimString Implementation Code #
##############################################
# Populate database with all ingredients
db = DictDatabase(CharacterNgramFeatureExtractor(2))
for ingredient in all_ingredients_final:
db.add(ingredient)
# Create searcher object to be used by candidates function
searcher = Searcher(db, CosineMeasure())
# Functions
def probability(word, N=sum(ingredients_count.values())):
"""
Returns the probability of the word appearing in the text
Usually, correctly spelled words will have a higher count and therefore probability than their mispellings
"""
return ingredients_count[word] / N
def candidates(word, searcher):
