class LanguageModelToy(Initializable):
"""
This takes the word embeddings from CompositionalLayerToyWithTables and creates sentence embeddings
Input is a 3d tensor with the dimensions of (num_words, num_subwords, batch_size) and
a 3d tensor a mask of size (num_words, num_subwords, batch_size)
All hidden state sizes are the same as the subword embedding size
This returns a 3d tensor with dimenstions of (num_words = num RNN states, batch_size, sentence embedding size)
"""
def __init__(self, batch_size, num_subwords, num_words, subword_embedding_size, input_vocab_size,
subword_RNN_hidden_state_size, LM_RNN_hidden_state_size, **kwargs):
super(LanguageModelToy, self).__init__(**kwargs)
self.batch_size = batch_size
self.num_subwords = num_subwords # number of subwords which make up a word
self.num_words = num_words # number of words in the sentence
self.subword_embedding_size = subword_embedding_size
self.input_vocab_size = input_vocab_size
self.subword_RNN_hidden_state_size = subword_RNN_hidden_state_size
self.LM_RNN_hidden_state_size = LM_RNN_hidden_state_size
self.compositional_layer = CompositionalLayerToyWithTables(self.batch_size, self.num_subwords, self.num_words,
self.subword_embedding_size, self.input_vocab_size,
self.subword_RNN_hidden_state_size, name='compositional_layer')
# has one RNN which reads the word embeddings into a sentence embedding
self.language_model_RNN = SimpleRecurrent(
dim=self.LM_RNN_hidden_state_size, activation=Identity(), name='language_model_RNN',
weights_init=Identity_init())
self.children = [self.compositional_layer, self.language_model_RNN]
@application(inputs=['subword_id_input_', 'subword_id_input_mask_'], outputs=['sentence_embeddings', 'sentence_embeddings_mask'])
def apply(self, subword_id_input_, subword_id_input_mask_):
"""
subword_id_input_ is a 3d tensor with the dimensions of shape = (num_words, num_subwords, batch_size).
It is expected as a dtype=uint16 or equivalent
subword_id_input_mask_ is a 3d tensor with the dimensions of shape = (num_words, num_subwords, batch_size).
It is expected as a dtype=uint8 or equivalent and has binary values of 1 when there is data and zero otherwise.
Returned is a 3d tensor of size (num_words = num RNN states, batch_size, sentence embedding size)
Also returned is a 1d tensor of size (batch_size) describing if the sentence is valid of empty in the batch
"""
word_embeddings, word_embeddings_mask = self.compositional_layer.apply(subword_id_input_, subword_id_input_mask_)
sentence_embeddings = self.language_model_RNN.apply(word_embeddings, mask=word_embeddings_mask)
sentence_embeddings_mask = word_embeddings_mask.max(axis=0).T
return sentence_embeddings, sentence_embeddings_mask
def __init__(self, batch_size, num_subwords, num_words, subword_embedding_size, input_vocab_size,
subword_RNN_hidden_state_size, LM_RNN_hidden_state_size, **kwargs):
super(LanguageModelToy, self).__init__(**kwargs)
self.batch_size = batch_size
self.num_subwords = num_subwords # number of subwords which make up a word
self.num_words = num_words # number of words in the sentence
self.subword_embedding_size = subword_embedding_size
self.input_vocab_size = input_vocab_size
self.subword_RNN_hidden_state_size = subword_RNN_hidden_state_size
self.LM_RNN_hidden_state_size = LM_RNN_hidden_state_size
self.compositional_layer = CompositionalLayerToyWithTables(self.batch_size, self.num_subwords, self.num_words,
self.subword_embedding_size, self.input_vocab_size,
self.subword_RNN_hidden_state_size, name='compositional_layer')
# has one RNN which reads the word embeddings into a sentence embedding
self.language_model_RNN = SimpleRecurrent(
dim=self.LM_RNN_hidden_state_size, activation=Identity(), name='language_model_RNN',
weights_init=Identity_init())
self.children = [self.compositional_layer, self.language_model_RNN]
