class UniversalSentenceEncoder: def __init__(self): super().__init__() model_version = 1 MODEL_PATH = "InferSent/encoder/infersent%s.pkl" % model_version W2V_PATH = 'InferSent/GloVe/glove.840B.300d.txt' if model_version == 1 else 'fastText/crawl-300d-2M.vec' params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': model_version } self.model = InferSent(params_model) self.model.load_state_dict(torch.load(MODEL_PATH)) self.model.eval() use_cuda = True self.model = self.model.cuda() if use_cuda else self.model self.model.set_w2v_path(W2V_PATH) self.model.build_vocab_k_words(K=100000) def semantic_sim(self, sents1, sents2): embed1 = self.model.encode(sents1, tokenize=False) embed2 = self.model.encode(sents2, tokenize=False) embed1 = torch.tensor(embed1) embed2 = torch.tensor(embed2) sts_encode1 = embed1 / torch.norm(embed1, p=2, dim=1, keepdim=True) sts_encode2 = embed2 / torch.norm(embed2, p=2, dim=1, keepdim=True) cosine_similarities = torch.sum(sts_encode1 * sts_encode2, dim=1) clip_cosine_similarities = torch.clamp(cosine_similarities, -1.0, 1.0) scores = 1.0 - torch.acos(clip_cosine_similarities) return scores.cpu().numpy()
def compute_intent_vectors(self, sentences): # TODO IMPLEMENT CACHING! from InferSent.models import InferSent infersent_folder = Path('./InferSent') infersent_path = Path(infersent_folder / 'encoder' / 'infersent1.pkl') MODEL_PARAMETERS = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': 1 } W2V_PATH = infersent_folder / 'GloVe' / 'glove.840B.300d.txt' model = InferSent(MODEL_PARAMETERS) model.load_state_dict(torch.load(infersent_path)) if torch.cuda.is_available(): model.cuda() model.set_w2v_path(W2V_PATH) model.build_vocab_k_words(K=100000) utterances_dict = self.get_utterances_dict(sentences) vectors = {} for i, (intent, sentences) in enumerate(utterances_dict.items()): LOGGER.info('{}/{} done'.format(i + 1, len(utterances_dict.items()))) embeddings = model.encode(sentences) avg_embedding = np.mean(embeddings, axis=0) vectors[intent] = avg_embedding return vectors
class _InferSent: def __init__(self): from InferSent.models import InferSent import torch V = 1 MODEL_PATH = 'encoder/infersent%s.pkl' % V params_model = { 'bsize': 256, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': V } self.infersent = InferSent(params_model) self.infersent.load_state_dict(torch.load(MODEL_PATH)) W2V_PATH = 'fastText/crawl-300d-2M.vec' self.infersent.set_w2v_path(W2V_PATH) def build_vocab(self, queries): self.infersent.build_vocab(queries, tokenize=True) def update_vocab(self, text): self.infersent.update_vocab(text, tokenize=True) def predict(self, text): # self.update_vocab(text) return self.infersent.encode(text, tokenize=True)
def inferSent(): import nltk # nltk.download('punkt') from InferSent.models import InferSent import torch # use_cuda = True device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # model = model.cuda() if use_cuda else model # V = 2 MODEL_PATH = 'encoder/infersent2.pkl' params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0, 'version': 2 } infersent = InferSent(params_model).to(device) infersent.load_state_dict(torch.load(MODEL_PATH)) W2V_PATH = 'fastText/crawl-300d-2M.vec' infersent.set_w2v_path(W2V_PATH) print('set w2v') infersent.build_vocab(dataset, tokenize=True) embeddings = infersent.encode(dataset, bsize=64, tokenize=True) idx = randint(0, len(dataset)) _, _ = infersent.visualize(dataset[idx]) print('done') return embeddings
def __init__(self, bsize=64, word_emb_dim=300, enc_lstm_dim=2048, pool_type='max', dpout_model=0.0, version=2, model_path='../infersent/infersent2.pkl', path_to_w2v='../fasttext/crawl-300d-2M.vec', use_cuda=True): self.version = version self.dpout_model = dpout_model self.pool_type = pool_type self.enc_lstm_dim = enc_lstm_dim self.word_emb_dim = word_emb_dim self.bsize = bsize model = InferSent({ 'bsize': bsize, 'word_emb_dim': word_emb_dim, 'enc_lstm_dim': enc_lstm_dim, 'pool_type': pool_type, 'dpout_model': dpout_model, 'version': version }) model.load_state_dict(torch.load(model_path)) model.set_w2v_path(path_to_w2v) if not use_cuda: self.model = model else: self.model = model.cuda() self.first_call = True
def _load_pretrained_model(verbose=True): if verbose: print(f">>> Loading pretrained model from {_MODEL_PATH}") infersent = InferSent(_PARAMS_MODEL) infersent.load_state_dict(torch.load(_MODEL_PATH)) infersent.set_w2v_path(_W2V_PATH) infersent.build_vocab_k_words(K=_K_WORDS_VOCAB) return infersent
def get_infersent(V=2): ''' Builds the infersent model using either GloVe or fastText ''' MODEL_PATH = 'encoder/infersent%s.pkl' %V if V == 2: W2V_PATH = 'fastText/crawl-300d-2M.vec' elif V == 1: W2V_PATH = 'GloVe/glove.840B.300d.txt' params_model = {'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, \ 'pool_type': 'max', 'dpout_model': 0.0, 'version': V} infersent = InferSent(params_model) infersent.load_state_dict(torch.load(MODEL_PATH)) infersent.set_w2v_path(W2V_PATH) return infersent
def __init__(self): #print("Initializing Infersent..") model_version = 1 MODEL_PATH = get_project_root() / Path("encoder/infersent%s.pkl" % model_version) params_model = {'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': model_version} model = InferSent(params_model) model.load_state_dict(torch.load(MODEL_PATH)) # word vector path for the model: W2V_PATH = get_project_root() / Path('GloVe/glove.840B.300d.txt' if model_version == 1 else '../fastText/crawl-300d-2M.vec') model.set_w2v_path(W2V_PATH) # build the vocabulary of word vectors model.build_vocab_k_words(K=100000) self.model = model
def Start_chatbot(): model_version = 1 MODEL_PATH = "../InferSent/encoder/infersent%s.pkl" % model_version params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': model_version } model = InferSent(params_model) model.load_state_dict(torch.load(MODEL_PATH)) use_cuda = False model = model.cuda() if use_cuda else model W2V_PATH = '../data/glove.6B.300d.txt' if model_version == 1 else '../dataset/fastText/crawl-300d-2M.vec' model.set_w2v_path(W2V_PATH) model.build_vocab_k_words(K=570000) dict = {} embeddings = {} questions = [] answers = [] with open('../data/questions.txt') as f: content = f.readlines() questions = [x.strip() for x in content] with open('../data/answers.txt') as f: content = f.readlines() answers = [x.strip() for x in content] for i in range(len(questions)): dict[questions[i]] = answers[i] embeddings[questions[i]] = model.encode([questions[i]])[0] return model, dict, embeddings
def generate_embeddings(df): paras = list(df["context"].drop_duplicates().reset_index(drop=True)) print("Paragraph count:", len(paras)) blob = TextBlob(" ".join(paras)) sentences = [item.raw for item in blob.sentences] params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': 2 } infersent = InferSent(params_model) infersent.load_state_dict(torch.load(infersent_pretrained_path)) infersent.set_w2v_path(glove_path) print("Building Infersent vocabulary") infersent.build_vocab(sentences, tokenize=True) dict_embeddings = {} print("Building sentence embeddings") print("Sentence count:", len(sentences)) for i in range(len(sentences)): dict_embeddings[sentences[i]] = infersent.encode([sentences[i]], tokenize=True) print("Building question embeddings") questions = df["question"].tolist() print("Questions count:", len(questions)) for i in range(len(questions)): dict_embeddings[questions[i]] = infersent.encode([questions[i]], tokenize=True) return dict_embeddings
import torch from InferSent.models import InferSent V = 2 MODEL_PATH = 'encoder/infersent%s.pkl' % V params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': V } infersent = InferSent(params_model) infersent.load_state_dict(torch.load(MODEL_PATH)) W2V_PATH = 'fastText/crawl-300d-2M.vec' infersent.set_w2v_path(W2V_PATH) sentences = [ "I am an engineer now.", "You can be an engineer.", "Building stuff is very fun.", "Stuff breaks often too though." ] infersent.build_vocab(sentences, tokenize=True) embeddings = infersent.encode(sentences, tokenize=True) infersent.visualize('A man plays an instrument.', tokenize=True) encoded_sentences = embeddings
def process(channel): # Load the Classifier tf.reset_default_graph() NN = classifer() NN.load('nn-classifier-v2') # Load the sentence embedder model_version = 1 MODEL_PATH = "InferSent/encoder/infersent%s.pkl" % model_version params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': model_version } model = InferSent(params_model) model.load_state_dict(torch.load(MODEL_PATH)) # Keep it on CPU or put it on GPU use_cuda = False model = model.cuda() if use_cuda else model # If infersent1 -> use GloVe embeddings. If infersent2 -> use InferSent embeddings. W2V_PATH = 'InferSent/GloVe/glove.840B.300d.txt' if model_version == 1 else 'InferSent/fastText/crawl-300d-2M.vec' model.set_w2v_path(W2V_PATH) # Load embeddings of K most frequent words model.build_vocab_k_words(K=100000) tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') all_files = glob.glob('../files/CableNews/%s/*.p' % channel) read_files = pickle.load(open('%s_visit.p' % (channel), 'rb')) counter = len(read_files) for file in tqdm(all_files): if file in read_files: continue else: read_files.append(file) if np.random.rand() < 0.3: pickle.dump(read_files, open('%s_visit.p' % (channel), 'wb')) res = pickle.load(open(file, 'rb')) results = {} prev_text = "" all_text = [] all_keys = [] for key in res.keys(): meta_data = res[key][0] # First in the list if len(meta_data['text']) < 10: continue # Make sure we drop the duplicates: Texts should be differents current_text = meta_data['text'][:10] if current_text == prev_text: continue else: prev_text = current_text text = tokenizer.tokenize(meta_data['text']) if len(text) <= 2: continue # Drop the first sentence text = text[1:] senteces = [] for s in text: #Drop super small and super large senteces if len(s.split()) > 30 and len(s.split()) < 50: senteces.append(s) if len(senteces) == 0: continue # Calculate the embedding all_text.extend(senteces) all_keys.extend([key] * len(senteces)) if len(all_text) == 0: continue all_embed = model.encode(all_text, bsize=128, tokenize=True, verbose=False) all_predictions = NN.predict(all_embed)[ 0] # Merge the probabilties and take top 2: prev_key = None total_prob = np.zeros((13, 1)) key_counter = 0 for current_key in all_keys: if current_key == prev_key: total_prob[:, 0] += all_predictions[key_counter, :] else: Topics = Ticks[np.flip(np.argsort(total_prob[:, 0])[-2:])] Probs = np.flip(np.sort(total_prob[:, 0])[-2:]) * 100 results[current_key] = { 'Topics': list(Topics), 'Probs': list(Probs), 'gender': res[current_key][0]['gender'], 'persons': res[current_key][0]['persons'], 'locations': res[current_key][0]['locations'] } prev_key = current_key total_prob = np.zeros((13, 1)) total_prob[:, 0] += all_predictions[key_counter, :] key_counter += 1 pickle.dump(results, open('processed_data/%s/%d.p' % (channel, counter), 'wb')) counter += 1
from InferSent.models import InferSent # In[110]: model_pkl = '../InferSent/encoder/infersent1.pkl' params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': 1 } infer_sent_model = InferSent(params_model) infer_sent_model.load_state_dict(torch.load(model_pkl)) # In[111]: infer_sent_model.set_w2v_path(glove_w2v_loc) infer_sent_model.build_vocab_k_words(K=100000) # infer_sent_model.to(torch.device("cuda:0")) # In[112]: infer_sent_model.encode(["This man is playing computer games"], tokenize=True) # In[113]:
import pickle from nltk.tokenize import sent_tokenize # Load Model from InferSent.models import InferSent MODEL_PATH = "encoder/infersent1.pkl" params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': 1 } model = InferSent(params_model) model.load_state_dict(torch.load(MODEL_PATH)) # CUDA use_cuda = True model = model.cuda() if use_cuda else model # If infersent1 -> use GloVe embeddings. If infersent2 -> use InferSent embeddings. W2V_PATH = 'GloVe/glove.840B.300d.txt' model.set_w2v_path(W2V_PATH) # Load embeddings of K most frequent words print('Load glove') model.build_vocab_k_words(K=2000000) # load sentence dis = '/home/shl183/nlp4note/classified_txt/discharge-sep'
class SentEnc(): def __init__(self, model_path, w2v_path, vocab_size=100000, device='cpu', hp=False): self.vocab = set() self.device = device self.hp = hp self.tokenizer = word_tokenize self.vocab_size = vocab_size params_model = { 'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': 1 } self.sent_model = InferSent(params_model).to(self.device).eval() self.sent_model.load_state_dict( torch.load(model_path, map_location=device)) if hp: self.sent_model = self.sent_model.half() # self.sent_model = self.sent_model.cuda() # self.sent_model.set_w2v_path(w2v_path) self.init_vocab(w2v_path) self.cache = dict() def init_vocab(self, w2v_path): self.w2v, self.bs, self.es = load_WE(w2v_path, self.vocab_size) self.vocab.update(self.w2v.keys()) # sentences = list(self.vocab) + [self.unk, self.bs, self.es] # self.sent_model.build_vocab(sentences, tokenize=True) def prune_sent(self, sent): if sent is None: sent_tok = [] else: sent_tok = self.tokenizer(sent) sent_tok = [x for x in sent_tok if x in self.w2v] sent_rec = ' '.join(sent_tok) return sent_rec def get_number_encoding(self, num): d = 256 device = 'cpu' # consider 4 decimal points a = int(num) b = int((num - a) * 1e2) a = str(a)[::-1] b = str(b) J = torch.arange(0, d) J_even = (J % 2 == 0).float().to(device) J = J.float().to(device) Ia = torch.arange(0, len(a)).float().to(device) Ib = (torch.arange(0, len(b)) + 1).float().to(device) A = torch.FloatTensor([float(x) for x in a]).to(device) B = torch.FloatTensor([float(x) for x in b]).to(device) J = J.float() A = torch.cat([A.view(1, -1)] * d, dim=0).T Ia = torch.cat([Ia.view(1, -1)] * d, dim=0).T B = torch.cat([B.view(1, -1)] * d, dim=0).T Ib = torch.cat([Ib.view(1, -1)] * d, dim=0).T resA = A * (2**Ia) / 10 * (J_even * torch.sin(Ia / (10000**(J / d))) + (1 - J_even) * torch.cos(Ia / (10000**( (J - 1) / d)))) resB = B * (2.0**(-Ib)) / 10 * ( J_even * torch.sin(-Ib / (10000**(J / d))) + (1 - J_even) * torch.cos(-Ib / (10000**((J - 1) / d)))) res = torch.sum(resA, axis=0) + torch.sum(resB, axis=0) res = res / (len(a) + len(b)) res = res.numpy() if d < 4096: res = np.append(res, np.zeros(4096 - d)).astype('float32') return res def is_number(self, string): # try: # num = abs(float(string)) # return num # except: # return None string = string.strip() if re.match('^[-+]?\d+\.\d+$', string): return abs(float(string)) if re.match('^[-+]?\d+$', string) and len(string) < 5: return abs(int(string)) if re.match('^[-+]?[\d,]+$', string) and len(string) < 5: return abs(int(string.replace(',', ''))) return None def get_text_encoding(self, sent): sent_tok = sent.split() s = np.array([self.w2v[self.bs]] + [self.w2v[x] for x in sent_tok] + [self.w2v[self.es]]) l = len(s) s = s.reshape([l, 1, 300]) s = torch.from_numpy(s).float() if self.hp: s = s.half() s = s.to(self.device) l = np.array([l], dtype='int64') # s = ' '.join(s) with torch.no_grad(): res = self.sent_model.forward((s, l)) # # v = torch.zeros([1,4096]) if res.shape[1] != 4096: print(v, s) v = res.cpu().detach().numpy()[0] del res return v def cache_sentences(self, sentences): self.sent_cache = set() self.num_cache = set() for s in sentences: if s is None: s = '' if REG: num = self.is_number(s) if num is not None: self.num_cache.add((num, s)) else: pruned_s = self.prune_sent(s) self.sent_cache.add(pruned_s) else: pruned_s = self.prune_sent(s) self.sent_cache.add(pruned_s) print(f'initialize {len(self.sent_cache)} text sentences...') for s in tqdm.tqdm(self.sent_cache): self.cache[s] = self.get_text_encoding(s) print(f'initialize {len(self.num_cache)} numeric sentences...') self.num_cache = list(self.num_cache) for n, n_str in tqdm.tqdm(self.num_cache): self.cache[n_str] = self.get_number_encoding(n) # p = Pool(NUM_THREADS) # vecs = p.map(self.get_number_encoding, self.num_cache) # p.terminate() # for n, v in zip(self.num_cache, vecs): # self.cache[str(num)] = v def __getitem__(self, sent): if sent is None: sent = '' if REG: num = self.is_number(sent) if num is not None: num_str = sent return self.cache[num_str] else: pruned_s = self.prune_sent(sent) return self.cache[pruned_s] pruned_s = self.prune_sent(sent) return self.cache[pruned_s]
'word_emb_dim': 300, 'enc_lstm_dim': 2048, 'pool_type': 'max', 'dpout_model': 0.0, 'version': VERSION } if VERSION == 1: W2V = 'C:/users/georg/Desktop/GloVe/glove.840B.300d.txt' else: W2V = 'C:/Users/georg/Desktop/fastText/crawl-300d-2M.vec' VOCAB_SIZE = 100000 NUM_STEPS = 300 # set up model model = InferSent(PARAMS).to(DEVICE) model.load_state_dict(torch.load(WEIGHTS)) model.set_w2v_path(W2V) word2vec = model.build_vocab_k_words(K=VOCAB_SIZE) # setup the NN-classifer vec2word = KNeighborsClassifier(n_neighbors=1) vecs = [] words = [] for key, val in word2vec.items(): if val.shape == (300, ): vecs.append(val) words.append(key) X = np.vstack(vecs) y = np.array(words) vec2word.fit(X, y)