class SentimentClassifier(keras.Model):
def __init__(self, vocab_size, embedding_size, lstm_units, num_classes):
super(SentimentClassifier, self).__init__()
self.emb = Embedding(input_dim=vocab_size,
output_dim=embedding_size,
mask_zero=True)
self.gauss = GaussianNoise(stddev=0.3)
self.dropout1 = Dropout(rate=0.3)
self.lstm1 = Bidirectional(
LSTM(return_sequences=True, units=lstm_units))
self.lstm2 = Bidirectional(LSTM(units=lstm_units))
self.dropout2 = Dropout(rate=0.4)
self.dense = Dense(units=num_classes, activation='sigmoid')
def call(self, x):
mask = self.emb.compute_mask(x)
x = self.emb(x)
x = self.gauss(x)
x = self.dropout1(x)
x = self.lstm1(x, mask=mask)
x = self.lstm2(x, mask=mask)
x = self.dropout2(x)
x = self.dense(x)
return x
class Encoder(tf.keras.Model):
""" Encoder model that compresses a given input sentences.
It recieves three embeddings as input. The word embeddings,
the part-of-speech embeddings, and the dependency relation
embeddings. A bidirectional LSTM is used to extract the
features that are used to predict whether a word is removed
or kept.
Args:
vocab_size: Vocabulary size defined by the training set.
enc_units: The number of units for the LSTM.
batch_size: The batch size.
"""
def __init__(self, vocab_size, enc_units, batch_sz):
super(Encoder, self).__init__()
self.batch_sz = batch_sz
self.enc_units = enc_units
self.word_embedding = Embedding(
WORD_DICT_SIZE,
WORD_EMB_DIM,
mask_zero=True,
embeddings_initializer='glorot_uniform')
self.pos_embedding = Embedding(POS_DICT_SIZE,
POS_EMB_DIM,
mask_zero=True,
embeddings_initializer='glorot_uniform')
self.dep_embedding = Embedding(DEP_DICT_SIZE,
DEP_EMB_DIM,
mask_zero=True,
embeddings_initializer='glorot_uniform')
self.lstm = LSTM(NUM_UNITS,
return_sequences=True,
return_state=True,
dropout=0.5,
recurrent_initializer='glorot_uniform')
self.bidirectional = Bidirectional(self.lstm, merge_mode='concat')
self.fc = Dense(1, activation='sigmoid')
def call(self, words, pos, dep):
word_emb = self.word_embedding(words)
pos_emb = self.pos_embedding(pos)
dep_emb = self.dep_embedding(dep)
mask = self.dep_embedding.compute_mask(words)
x = concatenate([word_emb, pos_emb, dep_emb])
x, forward_ouput, forward_state, backward_output, backward_state = self.bidirectional(
x, mask=mask)
output = tf.squeeze(self.fc(x), -1)
return output
class Encoder(tf.keras.Model):
''' Encoder model -- That takes a input sequence
returns encoder-outputs, encoder_final_state_h, encoder_final_state_c '''
def __init__(self, inp_vocab_size, embedding_size, lstm_size, input_length, qsn_matrix):
super().__init__()
self.vocab_size = inp_vocab_size
self.embedding_dim = embedding_size
self.lstm_units = lstm_size
self.input_length = input_length
self.qsn_matrix = qsn_matrix
self.enc_output = self.enc_state_h = self.enc_state_c = 0
#Initialize Embedding layer, output shape: (batch_size, input_length, embedding_dim)
self.embedding = Embedding(self.vocab_size, self.embedding_dim,
embeddings_initializer=tf.keras.initializers.Constant(self.qsn_matrix), trainable=False,
input_length=self.input_length, mask_zero=True, name="encoder_Embedding")
#Intialize Encoder LSTM layer
self.lstm = LSTM(units=self.lstm_units, activation='tanh', recurrent_activation='sigmoid',
kernel_initializer=tf.keras.initializers.glorot_uniform(seed=26),
recurrent_initializer=tf.keras.initializers.orthogonal(seed=54),
bias_initializer=tf.keras.initializers.zeros(), return_state=True, return_sequences=True, name="encoder_LSTM")
# Bidirectional layer
self.bidirectional = Bidirectional(self.lstm)
def call(self, input_sequence, states):
'''This function takes a sequence input and the initial states of the encoder.'''
# Embedding inputs, using pretrained glove vectors
embedded_input = self.embedding(input_sequence) # shape: (input_length, glove vector's dimension)
# mask for padding
mask = self.embedding.compute_mask(input_sequence)
# enc_out shape: (batch_size, input_length, lstm_size) & forward or backward h and c: (batch_size, lstm_size)
self.enc_out, enc_fw_state_h, enc_bw_state_h, enc_fw_state_c, enc_bw_state_c = self.bidirectional(embedded_input, mask=mask)
# Concatenating forward and backward states
self.enc_state_h = Concatenate()([enc_fw_state_h, enc_bw_state_h]) # enc_state_h and c shape: (batch_size, 2*lstm_size)
self.enc_state_c = Concatenate()([enc_fw_state_c, enc_bw_state_c])
return self.enc_out, self.enc_state_h, self.enc_state_c, mask
def initialize_states(self,batch_size):
'''Given a batch size it will return intial hidden state and intial cell state.'''
return (tf.zeros([batch_size, 2*self.lstm_units]), tf.zeros([batch_size, 2*self.lstm_units]))
class POSTagger(keras.Model):
def __init__(self):
super(POSTagger, self).__init__()
self.emb = Embedding(input_dim=len(vocab),
output_dim=300,
mask_zero=True,
weights=[data.embedding_matrix(vocab)])
self.lstm = Bidirectional(LSTM(300, return_sequences=True))
self.dense = Dense(17, activation='softmax')
def call(self, inputs):
x = self.emb(inputs)
mask = self.emb.compute_mask(inputs)
x = self.lstm(x, mask=mask)
output = self.dense(x)
return output
class SumEmbeddings(Layer):
def __init__(self, embeddings, input_length, trainable_embeddings=False):
super(SumEmbeddings, self).__init__()
self.vocab_size = embeddings.shape[0]
self.d_model = embeddings.shape[1]
self.emb_layer = Embedding(
input_dim=self.vocab_size,
output_dim=self.d_model,
embeddings_initializer=initializers.Constant(embeddings),
input_length=input_length,
trainable=trainable_embeddings,
mask_zero=True)
self.weight_emb_layer = Embedding(
input_dim=self.vocab_size,
output_dim=1,
embeddings_initializer=initializers.RandomNormal(1, 0.01),
input_length=input_length,
trainable=True,
mask_zero=True)
@tf.function
def call(self, inputs):
"""
Args:
inputs (tf.tensor): of shape (batch, seq_len, d_model)
mask (tf.tensor): of shape (batch, seq_len)
Returns:
a tensor of shape (batch, d_model)
"""
input_embs = self.emb_layer(inputs)
input_weights = self.weight_emb_layer(inputs)
out = tf.multiply(input_embs, input_weights)
mask = self.emb_layer.compute_mask(inputs)
if mask is not None:
mask = tf.expand_dims(tf.cast(mask, tf.float32), 2)
out = tf.multiply(out, mask)
return tf.reduce_sum(out, axis=1)
class PositionalEmbedding(layers.Layer):
'''
the model itself learns the positional embedding
'''
def __init__(self, vocab_size, max_len, embedding_dim=128, dp_rate=0.1):
super(PositionalEmbedding, self).__init__()
self.token_emb = Embedding(input_dim=vocab_size,
output_dim=embedding_dim,
mask_zero=True)
self.pos_emb = Embedding(input_dim=max_len,
output_dim=embedding_dim,
mask_zero=True)
self.dropout = Dropout(dp_rate)
def call(self, x, training):
positions = tf.range(start=0, limit=tf.shape(x)[1], delta=1)
positions = self.pos_emb(positions)
x = self.token_emb(x) + positions
return self.dropout(x, training)
def compute_mask(self, x, mask=None):
a = tf.cast(self.token_emb.compute_mask(x), tf.float32)
return 1 - a[:, tf.newaxis, :]