def get_dictionaries():
""" function to load all the dictionaries into the memory"""
queries = unicode(open('assets/csv/clean_query.txt').read(),
'utf-8').split('\n')
dictionary_fw = []
total_bigram = []
dictionary_name = 'dictionary'
fw_dictionary_name = 'fw_dictionary'
bigram_dictionary_name = 'bigram_dictionary'
dictionary = model.load_dictionary(dictionary_name)
fw_dictionary = model.load_dictionary(fw_dictionary_name)
bigram_dictionary = model.load_dictionary(bigram_dictionary_name)
if not (dictionary and fw_dictionary and bigram_dictionary):
for query in queries:
tokens = tokenize(query)
dictionary_fw.append(tokens[0])
dictionary_fw.append(lemmatize_tokens(tokens[0])[0])
query_bigram = list(nltk.bigrams(tokens)) + list(
nltk.bigrams(remove_stop_words(tokens))) + list(
nltk.bigrams(lemmatize_tokens(tokens))) + list(
nltk.bigrams(
lemmatize_tokens(remove_stop_words(tokens))))
for bigram in query_bigram:
total_bigram.append(' '.join(bigram))
all_tokens = tokenize(' '.join(queries))
dictionary = Counter(all_tokens) + Counter(
lemmatize_tokens(all_tokens))
fw_dictionary = Counter(dictionary_fw)
bigram_dictionary = Counter(total_bigram)
for key in dictionary.keys():
if dictionary[key] < 5:
del dictionary[key]
for key in bigram_dictionary.keys():
if bigram_dictionary[key] < 2:
del bigram_dictionary[key]
model.save_dictionary(dictionary_name, dictionary)
model.save_dictionary(fw_dictionary_name, fw_dictionary)
model.save_dictionary(bigram_dictionary_name, bigram_dictionary)
return dictionary, fw_dictionary, bigram_dictionary
def get_misspelled_synonyms():
SYNONYMS_DICTIONARY = 'synonyms'
synonyms_dic = model.load_dictionary(SYNONYMS_DICTIONARY)
if not synonyms_dic:
text = open('assets/csv/synonym.txt').read()
synonyms_dic = {}
rows = text.split('\n')
for row in rows:
row = row.split('=>')
if len(row) > 1:
word = row[1].strip()
synonyms = row[0].split(',')
for synonym in synonyms:
synonyms_dic[synonym.strip()] = word
model.save_dictionary(SYNONYMS_DICTIONARY, synonyms_dic)
return synonyms_dic
from fastapi import FastAPI
from pydantic import BaseModel
from utils import normalize_text
from model import load_model, load_dictionary
from read_config import read_env_var
app = FastAPI()
language = read_env_var("language")
model = load_model(language)
dictionary = load_dictionary(language)
print(f"Loading model with language {language}")
class Sentiment(BaseModel):
phrase: str
class Result(BaseModel):
phrase: str
language: str
positive: float
negative: float
@app.put("/api/analize", response_model=Result)
def get_users(sentiment: Sentiment):
result = model.predict([normalize_text(dictionary, sentiment.phrase)])
return Result(phrase=sentiment.phrase, positive=result[0][1], negative=result[0][0], language=language)
padding = (int((im.shape[0] - im.shape[1]) / 2), 0)
else:
padding = (0, int((im.shape[1] - im.shape[0]) / 2))
data_transforms = transforms.Compose([
transforms.ToPILImage(),
transforms.Pad(padding, fill=0),
])
pim = data_transforms(im)
return (pim)
densenet = DenseNet121(14)
# densenet = addDropout(densenet, p=0)
saved_model_path = "../Models/model.pth.tar"
densenet.load_state_dict(load_dictionary(saved_model_path, map_location='cpu'))
if torch.cuda.is_available():
densenet = densenet.cuda()
name = "Exp_64_512_0.00001_RandomLabel_4.0"
#Path to the experiment (it would be the github)
ExpDir = "/media/vince/MILA/ChestXrays/ALI/model/" + name
ExpDir = "/network/home/frappivi/ChestXrays/ALI/model/" + name
isize = 64
LS = 512 #Latent Space Size
ColorsNumber = 1 #Number of color (always 1 for x-ray)
isize = 64
#Load model
CP = -2 #Checkpoint to load (-2 for latest one, -1 for last epoch)
DisX, DisZ, DisXZ, GenZ, GenX, CP, DiscriminatorLoss, AllAUCs = GenModel(