def make_lang(train_x,dev_x,test_x):
lang = Lang('essays')
train_words = []
for essay in train_x:
if params['padding_level']=='document':
train_words += essay
elif params['padding_level']=='sentence':
for sent in essay:
train_words += sent
lang.build_vocab(train_words, voc_size=params['voc_size'])
params['voc_size'] = lang.n_words
lang.make_embeddings(emb_size=params['emb_size'], emb_type=params['emb_type'])
return lang
def make_lang(texts):
lang = Lang('texts')
words = []
for text in texts:
for sent in text:
words += sent
lang.build_vocab(words, voc_size=params['voc_size'])
params['voc_size'] = lang.n_words
lang.make_embeddings(emb_size=params['emb_size'],
emb_type=params['emb_type'])
return lang
