def inferSent():
import nltk
# nltk.download('punkt')
from InferSent.models import InferSent
import torch
# use_cuda = True
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# model = model.cuda() if use_cuda else model
# V = 2
MODEL_PATH = 'encoder/infersent2.pkl'
params_model = {
'bsize': 64,
'word_emb_dim': 300,
'enc_lstm_dim': 2048,
'pool_type': 'max',
'dpout_model': 0,
'version': 2
}
infersent = InferSent(params_model).to(device)
infersent.load_state_dict(torch.load(MODEL_PATH))
W2V_PATH = 'fastText/crawl-300d-2M.vec'
infersent.set_w2v_path(W2V_PATH)
print('set w2v')
infersent.build_vocab(dataset, tokenize=True)
embeddings = infersent.encode(dataset, bsize=64, tokenize=True)
idx = randint(0, len(dataset))
_, _ = infersent.visualize(dataset[idx])
print('done')
return embeddings
'version': V
}
infersent = InferSent(params_model)
infersent.load_state_dict(torch.load(MODEL_PATH))
W2V_PATH = 'fastText/crawl-300d-2M.vec'
infersent.set_w2v_path(W2V_PATH)
sentences = [
"I am an engineer now.", "You can be an engineer.",
"Building stuff is very fun.", "Stuff breaks often too though."
]
infersent.build_vocab(sentences, tokenize=True)
embeddings = infersent.encode(sentences, tokenize=True)
infersent.visualize('A man plays an instrument.', tokenize=True)
encoded_sentences = embeddings
# greedy decoder
def greedy_decoder(data):
# index for largest probability each row
return [np.argmax(s) for s in data]
# decode sequence
result = greedy_decoder(encoded_sentences)
print(result)