class LanguageModelToy(Initializable): """ This takes the word embeddings from CompositionalLayerToyWithTables and creates sentence embeddings Input is a 3d tensor with the dimensions of (num_words, num_subwords, batch_size) and a 3d tensor a mask of size (num_words, num_subwords, batch_size) All hidden state sizes are the same as the subword embedding size This returns a 3d tensor with dimenstions of (num_words = num RNN states, batch_size, sentence embedding size) """ def __init__(self, batch_size, num_subwords, num_words, subword_embedding_size, input_vocab_size, subword_RNN_hidden_state_size, LM_RNN_hidden_state_size, **kwargs): super(LanguageModelToy, self).__init__(**kwargs) self.batch_size = batch_size self.num_subwords = num_subwords # number of subwords which make up a word self.num_words = num_words # number of words in the sentence self.subword_embedding_size = subword_embedding_size self.input_vocab_size = input_vocab_size self.subword_RNN_hidden_state_size = subword_RNN_hidden_state_size self.LM_RNN_hidden_state_size = LM_RNN_hidden_state_size self.compositional_layer = CompositionalLayerToyWithTables(self.batch_size, self.num_subwords, self.num_words, self.subword_embedding_size, self.input_vocab_size, self.subword_RNN_hidden_state_size, name='compositional_layer') # has one RNN which reads the word embeddings into a sentence embedding self.language_model_RNN = SimpleRecurrent( dim=self.LM_RNN_hidden_state_size, activation=Identity(), name='language_model_RNN', weights_init=Identity_init()) self.children = [self.compositional_layer, self.language_model_RNN] @application(inputs=['subword_id_input_', 'subword_id_input_mask_'], outputs=['sentence_embeddings', 'sentence_embeddings_mask']) def apply(self, subword_id_input_, subword_id_input_mask_): """ subword_id_input_ is a 3d tensor with the dimensions of shape = (num_words, num_subwords, batch_size). It is expected as a dtype=uint16 or equivalent subword_id_input_mask_ is a 3d tensor with the dimensions of shape = (num_words, num_subwords, batch_size). It is expected as a dtype=uint8 or equivalent and has binary values of 1 when there is data and zero otherwise. Returned is a 3d tensor of size (num_words = num RNN states, batch_size, sentence embedding size) Also returned is a 1d tensor of size (batch_size) describing if the sentence is valid of empty in the batch """ word_embeddings, word_embeddings_mask = self.compositional_layer.apply(subword_id_input_, subword_id_input_mask_) sentence_embeddings = self.language_model_RNN.apply(word_embeddings, mask=word_embeddings_mask) sentence_embeddings_mask = word_embeddings_mask.max(axis=0).T return sentence_embeddings, sentence_embeddings_mask