def conc_emb_projection_dropout_Bi_LSTM_conc_emb_attention_projection(embeddings, classes, max_length):
''' BASELINE 8
A simple model having a concatenation of word and aspect embeddings with gaussian
noise,a projection layer with a tanh activation , a dropout layer,a Bi-LSTM cell,
a concatenation of each hidden state with an aspect embedding, attention on top,
a final dropout layer,a projection layer with a relu activation and a softmax for
the final output'''
rnn_size = 128
# create sent embeddings
sent_input = Input(shape=(max_length,), dtype='int32', name='sentence')
embed = embedding_layer(embeddings=embeddings, max_length=max_length
, trainable=False)(sent_input)
# create aspect embeddings
auxilary_input = Input(shape=(1,), dtype='int32', name='auxilary_input')
aspect_embed = embedding_layer2(embeddings=embeddings, trainable=True)(auxilary_input)
# bring the aspect embeddings in the appropriate format (we need to repeat the same aspect embedding k times)
aspect_embed = keras.layers.Flatten(name='auxilary_flattened')(aspect_embed)
aspect_embed = keras.layers.RepeatVector(max_length, name='auxilary_repeated')(aspect_embed)
# concatenate the aspect embedding with the sent embedding
concat = keras.layers.concatenate([embed, aspect_embed], name='concatenated_embeddings')
concat = keras.layers.GaussianNoise(0.2)(concat)
concat = keras.layers.Dropout(0.2, noise_shape=(1, 350), name='dropout_concat_emb', )(concat)
concat = keras.layers.Dense(1024, activation='tanh', name='dense_after_emb')(concat)
lstm = Bidirectional(LSTM(rnn_size, return_sequences=True))(concat)
hidden = keras.layers.concatenate([lstm, aspect_embed], name='concatenated_lstm_embedding')
attention = Attention(name='attention')(hidden)
atten_dropout = keras.layers.Dropout(0.2, noise_shape=(1, 306))(attention)
output = Dense(256, activation='relu', name='dense_relu')(atten_dropout)
output = Dense(classes, activation='softmax', name='dense_softmax')(output)
model = Functional_Model(inputs=[sent_input, auxilary_input], outputs=output)
model.compile(optimizer=rmsprop(), loss=categorical_crossentropy, metrics=['accuracy'])
print(model.summary())
return model
def emd_conc_asp_emb_LSTM_attention(embeddings, classes, max_length):
''' BASELINE 3
A simple model having a concatenation of word and aspect embeddings,a LSTM
cell after that and attention on top and a softmax for the final output'''
rnn_size = 128
# create sent embeddings
sent_input = Input(shape=(max_length,), dtype='int32', name='sentence')
embed = embedding_layer(embeddings=embeddings, max_length=max_length
, trainable=False)(sent_input)
# create aspect embeddings
auxilary_input = Input(shape=(1, ), dtype='int32', name='auxilary_input')
aspect_embed = embedding_layer2(embeddings=embeddings, trainable=True)(auxilary_input)
# bring the aspect embeddings in the appropriate format (we need to repeat the same aspect embedding k times)
aspect_embed = keras.layers.Flatten(name='auxilary_flattened')(aspect_embed)
aspect_embed = keras.layers.RepeatVector(max_length, name='auxilary_repeated')(aspect_embed)
# concatenate the aspect embedding with the sent embedding
concat = keras.layers.concatenate([embed, aspect_embed], name='concatenated_embeddings')
lstm = LSTM(rnn_size, return_sequences=True)(concat)
attention = Attention(name='attention')(lstm)
output = Dense(classes, activation='softmax', name='dense_softmax')(attention)
model = Functional_Model(inputs=[sent_input, auxilary_input], outputs=output)
model.compile(optimizer=rmsprop(), loss=categorical_crossentropy, metrics=['accuracy'])
print(model.summary())
return model
def rnn_entities_attributes5(embeddings, max_length):
def embedding_layer(embeddings,
max_length,
trainable=False,
masking=False):
vocab_size = embeddings.shape[0]
embedding_size = embeddings.shape[1]
emb_layer = Embedding(input_dim=vocab_size,
output_dim=embedding_size,
weights=[embeddings],
input_length=max_length,
mask_zero=masking,
trainable=trainable)
return emb_layer
########################################################
# Optional Parameters
########################################################
rnn_size = 128
trainable_emb = False
########################################################
inputs = Input(shape=(max_length, ), dtype='int32')
embed = embedding_layer(embeddings=embeddings,
max_length=max_length,
trainable=trainable_emb)(inputs)
embed_noise = gausian_noise({{choice([0.2, 0.3])}})(embed)
embed_dropout = dropout({{choice([0.2, 0.3, 0.4, 0.5])}})(embed_noise)
dense_emb = Dense({{choice([128, 256, 512, 1024])}},
name='dense_after_emb')(embed_dropout)
lstm_left, state_h_left, state_c_left = LSTM(
rnn_size, return_sequences=True, return_state=True)(dense_emb)
lstm_right, state_h_right, state_c_rigt = LSTM(
rnn_size,
return_sequences=True,
return_state=True,
go_backwards=True)(dense_emb)
lstm = keras.layers.concatenate([lstm_left, lstm_right])
lstm_dropout = dropout({{choice([0.2, 0.3, 0.4, 0.5])}})(lstm)
atten = Attention()(lstm_dropout)
output1 = Dense(6, activation='sigmoid')(atten)
output1_l1 = keras.layers.Lambda(lambda x: K.expand_dims(x, axis=-1))(
output1)
output2 = Dense(5, activation='sigmoid')(atten)
output2_l2 = keras.layers.Lambda(lambda x: K.expand_dims(x, axis=-1))(
output2)
dot = keras.layers.Dot(axes=2)([output2_l2, output1_l1])
flatten = keras.layers.Flatten()(dot)
concat = keras.layers.concatenate([flatten, atten])
output3 = Dense(256, activation='relu')(concat)
output3 = Dense(12, activation='sigmoid')(output3)
model = Functional_Model(inputs=inputs, outputs=output3)
model.compile(optimizer=rmsprop(),
loss=binary_crossentropy,
metrics=[f1])
print(model.summary())
return model
def entities_attributes_attention_dense(embeddings, max_length):
''' BASELINE #1
This a simple baseline model using an embedding layer ,
afterwards an attention layer and on top a simple dense
layer with a sigmoid to take a decision for all the E#A pairs '''
inputs = Input(shape=(max_length, ), dtype='int32')
embed = embedding_layer(embeddings=embeddings, max_length=max_length, trainable=False, masking=False)(inputs)
atten = Attention()(embed)
output = Dense(13, activation='sigmoid')(atten)
model = Functional_Model(inputs=inputs, outputs=output)
model.compile(optimizer=rmsprop(), loss=binary_crossentropy, metrics=[f1])
return model
def embeddings_attention(embeddings, classes, max_length):
''' BASELINE 1
A simple model having an embedding layer with attention on top and a softmax for the final output'''
# create sent embeddings
sent_input = Input(shape=(max_length,), dtype='int32', name='sentence')
embed = embedding_layer(embeddings=embeddings, max_length=max_length
, trainable=False)(sent_input)
attention = Attention(name='attention')(embed)
output = Dense(classes, activation='softmax', name='dense_softmax')(attention)
model = Functional_Model(inputs=sent_input, outputs=output)
model.compile(optimizer=Adam(), loss=categorical_crossentropy, metrics=['accuracy'])
print(model.summary())
return model
def entities_attributes_LSTM_attention_dence_dropout(embeddings, max_length):
''' BASELINE 3
This is a model that uses embedding layer,a dropout layer, afterwards a LSTM layer with a
self attention mechanism on top, a dropout layer and finally a dense layer
to make the decision'''
inputs = Input(shape=(max_length,), dtype='int32')
embed = embedding_layer(embeddings=embeddings, max_length=max_length, trainable=False, masking=False)(inputs)
embed_noise = keras.layers.GaussianNoise(0.2)(embed)
embed_dropout = keras.layers.Dropout(0.3, noise_shape=(1, 300))(embed_noise)
lstm = LSTM(units=128, return_sequences=True)(embed_dropout)
atten = Attention()(lstm)
atten_dropout = keras.layers.Dropout(0.2, noise_shape=(1, 128))(atten)
output = Dense(13, activation='sigmoid')(atten_dropout)
model = Functional_Model(inputs=inputs, outputs=output)
model.compile(optimizer=rmsprop(), loss=binary_crossentropy, metrics=[f1])
return model
def entities_attributes_projection_BiLSTM_attention_dencex2_dropout(embeddings, max_length):
''' BASELINE 6
This is a model that uses embedding layer, a projection of the embeddings , a dropout layer,
afterwards a Bi_LSTM layer with a self attention mechanism on top, a dropout layer and finally
a 2 dense layers (one with activation relu) to make the decision'''
inputs = Input(shape=(max_length,), dtype='int32')
embed = embedding_layer(embeddings=embeddings, max_length=max_length, trainable=False, masking=False)(inputs)
embed_noise = keras.layers.GaussianNoise(0.2)(embed)
embed_dropout = keras.layers.Dropout(0.3, noise_shape=(1, 300))(embed_noise)
dense_emb = keras.layers.Dense(1024, activation='tanh', name='dense_after_emb')(embed_dropout)
lstm = keras.layers.Bidirectional(LSTM(units=128, return_sequences=True))(dense_emb)
atten = Attention()(lstm)
atten_dropout = keras.layers.Dropout(0.2, noise_shape=(1, 256))(atten)
output = Dense(250, activation='relu')(atten_dropout)
output = Dense(13, activation='sigmoid')(output)
model = Functional_Model(inputs=inputs, outputs=output)
model.compile(optimizer=rmsprop(), loss=binary_crossentropy, metrics=[f1])
return model
def aspect_polarity(embeddings, classes, max_length, X_train, train_aux,
sentiment_intensity_train, y_train, X_test, gold_aux,
sentiment_intensity_test, y_test):
rnn_size = 128
max_length = 80
vocab_size = embeddings.shape[0]
embedding_size = embeddings.shape[1]
emb_layer = Embedding(input_dim=vocab_size,
output_dim=embedding_size,
weights=[embeddings],
input_length=max_length,
mask_zero=False,
trainable=False,
name='word_embedding')
vocab_size = 13
embedding_size = embeddings.shape[1]
emb_layer2 = Embedding(input_dim=vocab_size,
output_dim=embedding_size,
input_length=1,
mask_zero=False,
trainable=True,
name='aspect_embedding')
sent_input = Input(shape=(max_length, ), dtype='int32', name='sentence')
embed = emb_layer(sent_input)
intensity_input = Input(shape=(3, ),
dtype='float32',
name='setiment_intesity')
auxilary_input = Input(shape=(1, ), dtype='int32', name='auxilary_input')
aspect_embed = emb_layer2(auxilary_input)
aspect_embed = keras.layers.Flatten(
name='auxilary_flattened')(aspect_embed)
aspect_embed = keras.layers.RepeatVector(
max_length, name='auxilary_repeated')(aspect_embed)
concat = keras.layers.concatenate([embed, aspect_embed],
name='concatenated_embeddings')
concat = keras.layers.GaussianNoise({{uniform(0, 1)}},
name='gausian_noise')(concat)
concat = keras.layers.Dropout({{uniform(0, 1)}},
name='dropout_concat_emb')(concat)
lstm, state_h, state_c = LSTM({{choice([64, 128])}},
return_sequences=True,
return_state=True,
name='lstm_layer')(concat)
lstm = keras.layers.Dropout({{uniform(0, 1)}})(lstm)
hidden = keras.layers.concatenate([lstm, aspect_embed],
name='concatenated_lstm_embedding')
attention = Attention(name='attention')(hidden)
state_c = keras.layers.concatenate([state_c, intensity_input])
atten_final_state = keras.layers.concatenate([attention, state_c])
output = Dense(units={{choice([50, 150, 250])}},
activation={{choice(['relu', 'tanh'])}},
name='dense_relu')(atten_final_state)
output = Dense(classes, activation='softmax', name='dense_softmax')(output)
model = Functional_Model(
inputs=[sent_input, auxilary_input, intensity_input], outputs=output)
model.compile(optimizer={{choice(['adam', 'rmsprop'])}},
loss=categorical_crossentropy,
metrics=['accuracy'])
print(model.summary())
model.fit([X_train, train_aux, sentiment_intensity_train],
y_train,
epochs=1,
batch_size={{choice([64, 128])}},
validation_data=([X_test, gold_aux,
sentiment_intensity_test], y_test))
scores = model.predict([X_test, gold_aux, sentiment_intensity_test])
threshhold = {{uniform(0, 1)}}
scores[scores > threshhold] = 1
scores[scores < (1 - threshhold)] = 0
acc = accuracy2(y_test, scores)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def build(self,
number_of_periods=None,
period_length=7,
batch_size=1,
loss="mse"):
"""
Builds an LSTM model using Keras. This function
works as a simple wrapper for a manually created
model.
Parameters
----------
period_length: int
The size of each observation used as input.
number_of_periods: int, default None
The number of periods available in the
dataset. If None, the model will be built
using all available periods - 1 (used for validation).
batch_size: int
The size of the batch used in each training
period.
Returns
-------
model: Keras model
Compiled Keras model that can be trained
and stored in disk.
"""
if not number_of_periods:
number_of_periods = self.default_number_of_periods
if self.model_type == 'sequential':
self.model = Sequential()
self.model.add(
LSTM(units=period_length,
batch_input_shape=(batch_size, number_of_periods,
period_length),
input_shape=(number_of_periods, period_length),
return_sequences=False,
stateful=False))
self.model.add(Dense(units=period_length))
self.model.add(Activation("linear"))
self.model.compile(loss=loss, optimizer="rmsprop")
else:
input = Input(shape=(number_of_periods, period_length))
x = LSTM(units=period_length,
batch_input_shape=(batch_size, number_of_periods,
period_length),
input_shape=(number_of_periods, period_length),
return_sequences=False,
stateful=False)(input)
x0 = Dense(units=period_length, activation='linear')(x)
x1 = Dense(units=period_length, activation='linear')(x)
x2 = Dense(units=period_length, activation='linear')(x)
self.model = Functional_Model(input, [x0, x1, x2])
self.model.compile(loss=loss, optimizer="rmsprop")
return self.model
