class SentimentClassifier(keras.Model): def __init__(self, vocab_size, embedding_size, lstm_units, num_classes): super(SentimentClassifier, self).__init__() self.emb = Embedding(input_dim=vocab_size, output_dim=embedding_size, mask_zero=True) self.gauss = GaussianNoise(stddev=0.3) self.dropout1 = Dropout(rate=0.3) self.lstm1 = Bidirectional( LSTM(return_sequences=True, units=lstm_units)) self.lstm2 = Bidirectional(LSTM(units=lstm_units)) self.dropout2 = Dropout(rate=0.4) self.dense = Dense(units=num_classes, activation='sigmoid') def call(self, x): mask = self.emb.compute_mask(x) x = self.emb(x) x = self.gauss(x) x = self.dropout1(x) x = self.lstm1(x, mask=mask) x = self.lstm2(x, mask=mask) x = self.dropout2(x) x = self.dense(x) return x
class Encoder(tf.keras.Model): """ Encoder model that compresses a given input sentences. It recieves three embeddings as input. The word embeddings, the part-of-speech embeddings, and the dependency relation embeddings. A bidirectional LSTM is used to extract the features that are used to predict whether a word is removed or kept. Args: vocab_size: Vocabulary size defined by the training set. enc_units: The number of units for the LSTM. batch_size: The batch size. """ def __init__(self, vocab_size, enc_units, batch_sz): super(Encoder, self).__init__() self.batch_sz = batch_sz self.enc_units = enc_units self.word_embedding = Embedding( WORD_DICT_SIZE, WORD_EMB_DIM, mask_zero=True, embeddings_initializer='glorot_uniform') self.pos_embedding = Embedding(POS_DICT_SIZE, POS_EMB_DIM, mask_zero=True, embeddings_initializer='glorot_uniform') self.dep_embedding = Embedding(DEP_DICT_SIZE, DEP_EMB_DIM, mask_zero=True, embeddings_initializer='glorot_uniform') self.lstm = LSTM(NUM_UNITS, return_sequences=True, return_state=True, dropout=0.5, recurrent_initializer='glorot_uniform') self.bidirectional = Bidirectional(self.lstm, merge_mode='concat') self.fc = Dense(1, activation='sigmoid') def call(self, words, pos, dep): word_emb = self.word_embedding(words) pos_emb = self.pos_embedding(pos) dep_emb = self.dep_embedding(dep) mask = self.dep_embedding.compute_mask(words) x = concatenate([word_emb, pos_emb, dep_emb]) x, forward_ouput, forward_state, backward_output, backward_state = self.bidirectional( x, mask=mask) output = tf.squeeze(self.fc(x), -1) return output
class Encoder(tf.keras.Model): ''' Encoder model -- That takes a input sequence returns encoder-outputs, encoder_final_state_h, encoder_final_state_c ''' def __init__(self, inp_vocab_size, embedding_size, lstm_size, input_length, qsn_matrix): super().__init__() self.vocab_size = inp_vocab_size self.embedding_dim = embedding_size self.lstm_units = lstm_size self.input_length = input_length self.qsn_matrix = qsn_matrix self.enc_output = self.enc_state_h = self.enc_state_c = 0 #Initialize Embedding layer, output shape: (batch_size, input_length, embedding_dim) self.embedding = Embedding(self.vocab_size, self.embedding_dim, embeddings_initializer=tf.keras.initializers.Constant(self.qsn_matrix), trainable=False, input_length=self.input_length, mask_zero=True, name="encoder_Embedding") #Intialize Encoder LSTM layer self.lstm = LSTM(units=self.lstm_units, activation='tanh', recurrent_activation='sigmoid', kernel_initializer=tf.keras.initializers.glorot_uniform(seed=26), recurrent_initializer=tf.keras.initializers.orthogonal(seed=54), bias_initializer=tf.keras.initializers.zeros(), return_state=True, return_sequences=True, name="encoder_LSTM") # Bidirectional layer self.bidirectional = Bidirectional(self.lstm) def call(self, input_sequence, states): '''This function takes a sequence input and the initial states of the encoder.''' # Embedding inputs, using pretrained glove vectors embedded_input = self.embedding(input_sequence) # shape: (input_length, glove vector's dimension) # mask for padding mask = self.embedding.compute_mask(input_sequence) # enc_out shape: (batch_size, input_length, lstm_size) & forward or backward h and c: (batch_size, lstm_size) self.enc_out, enc_fw_state_h, enc_bw_state_h, enc_fw_state_c, enc_bw_state_c = self.bidirectional(embedded_input, mask=mask) # Concatenating forward and backward states self.enc_state_h = Concatenate()([enc_fw_state_h, enc_bw_state_h]) # enc_state_h and c shape: (batch_size, 2*lstm_size) self.enc_state_c = Concatenate()([enc_fw_state_c, enc_bw_state_c]) return self.enc_out, self.enc_state_h, self.enc_state_c, mask def initialize_states(self,batch_size): '''Given a batch size it will return intial hidden state and intial cell state.''' return (tf.zeros([batch_size, 2*self.lstm_units]), tf.zeros([batch_size, 2*self.lstm_units]))
class POSTagger(keras.Model): def __init__(self): super(POSTagger, self).__init__() self.emb = Embedding(input_dim=len(vocab), output_dim=300, mask_zero=True, weights=[data.embedding_matrix(vocab)]) self.lstm = Bidirectional(LSTM(300, return_sequences=True)) self.dense = Dense(17, activation='softmax') def call(self, inputs): x = self.emb(inputs) mask = self.emb.compute_mask(inputs) x = self.lstm(x, mask=mask) output = self.dense(x) return output
class SumEmbeddings(Layer): def __init__(self, embeddings, input_length, trainable_embeddings=False): super(SumEmbeddings, self).__init__() self.vocab_size = embeddings.shape[0] self.d_model = embeddings.shape[1] self.emb_layer = Embedding( input_dim=self.vocab_size, output_dim=self.d_model, embeddings_initializer=initializers.Constant(embeddings), input_length=input_length, trainable=trainable_embeddings, mask_zero=True) self.weight_emb_layer = Embedding( input_dim=self.vocab_size, output_dim=1, embeddings_initializer=initializers.RandomNormal(1, 0.01), input_length=input_length, trainable=True, mask_zero=True) @tf.function def call(self, inputs): """ Args: inputs (tf.tensor): of shape (batch, seq_len, d_model) mask (tf.tensor): of shape (batch, seq_len) Returns: a tensor of shape (batch, d_model) """ input_embs = self.emb_layer(inputs) input_weights = self.weight_emb_layer(inputs) out = tf.multiply(input_embs, input_weights) mask = self.emb_layer.compute_mask(inputs) if mask is not None: mask = tf.expand_dims(tf.cast(mask, tf.float32), 2) out = tf.multiply(out, mask) return tf.reduce_sum(out, axis=1)
class PositionalEmbedding(layers.Layer): ''' the model itself learns the positional embedding ''' def __init__(self, vocab_size, max_len, embedding_dim=128, dp_rate=0.1): super(PositionalEmbedding, self).__init__() self.token_emb = Embedding(input_dim=vocab_size, output_dim=embedding_dim, mask_zero=True) self.pos_emb = Embedding(input_dim=max_len, output_dim=embedding_dim, mask_zero=True) self.dropout = Dropout(dp_rate) def call(self, x, training): positions = tf.range(start=0, limit=tf.shape(x)[1], delta=1) positions = self.pos_emb(positions) x = self.token_emb(x) + positions return self.dropout(x, training) def compute_mask(self, x, mask=None): a = tf.cast(self.token_emb.compute_mask(x), tf.float32) return 1 - a[:, tf.newaxis, :]