def GRUattention(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
if params['bidirectional']:
x = Bidirectional(CuDNNGRU(params['lstm_units'],
return_sequences=True))(embed_input_)
else:
x = CuDNNGRU(params['lstm_units'], return_sequences=True)(embed_input_)
x = AttentionWithContext()(x)
# x = GlobalAveragePooling1D()(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(256, activation='relu')(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def GRUHierarchical2(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
l_lstm = Bidirectional(
CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_)
l_dense = TimeDistributed(Dense(256))(l_lstm)
l_att = GlobalMaxPooling1D()(l_dense)
sentEncoder = Model(input_, l_att)
review_input = Input(shape=(1, params['sequence_length']))
review_encoder = TimeDistributed(sentEncoder)(review_input)
l_lstm_sent = Bidirectional(
CuDNNGRU(params['lstm_units'], return_sequences=True))(review_encoder)
l_dense_sent = TimeDistributed(Dense(256))(l_lstm_sent)
l_att = GlobalMaxPooling1D()(l_dense_sent)
l_att3 = AttentionWithContext()(l_dense_sent)
merge_layer2 = concatenate([l_att, l_att3])
x = Dense(6, activation='sigmoid')(merge_layer2)
model = Model(inputs=review_input, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def GRUConvdeep3V2(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
x = Activation('tanh')(embed_input_)
x = SpatialDropout1D(0.1, name='embed_drop')(x)
if params['bidirectional']:
x_g1 = Bidirectional(
CuDNNGRU(params['lstm_units'],
return_sequences=True))(x)
x_l1 = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True))(x_g1)
x_g2 = Bidirectional(
CuDNNGRU(params['lstm_units'],
return_sequences=True))(x_l1)
x1 = GlobalMaxPooling1D()(x_g2)
x2 = GlobalAveragePooling1D()(x_g2)
x3 = AttentionWithContext()(x_g2)
merge_layer = concatenate([x1, x2, x3])
x_conv = Conv1D(64, kernel_size=3, padding='valid',
kernel_initializer='glorot_uniform')(x_g2)
x1_conv = GlobalMaxPooling1D()(x_conv)
x2_conv = GlobalAveragePooling1D()(x_conv)
x3_conv = AttentionWithContext()(x_conv)
merge_layer2 = concatenate([x1, x2, x3, x1_conv, x2_conv, x3_conv])
x = Dropout(params['dropout_rate'])(merge_layer2)
x = Dense(256, activation='relu')(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def LSTMdeep2(params):
embed_dropout_rate = 0.1
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
x = Activation('tanh')(embed_input_)
x = SpatialDropout1D(embed_dropout_rate, name='embed_drop')(x)
if params['bidirectional']:
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform'))(x)
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform'))(x)
x1 = GlobalMaxPooling1D()(x)
x2 = GlobalAveragePooling1D()(x)
x3 = AttentionWithContext()(x)
else:
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform')(x)
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True,
kernel_initializer='he_uniform')(x)
x = GlobalMaxPooling1D()(x)
merge_layer = concatenate([x1, x2, x3])
x = Dropout(params['dropout_rate'])(merge_layer)
x = Dense(256)(x)
x = PReLU()(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def LSTMattentionBranched(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
if params['bidirectional']:
x = Bidirectional(
CuDNNLSTM(params['lstm_units'],
return_sequences=True))(embed_input_)
else:
x = CuDNNLSTM(params['lstm_units'],
return_sequences=True)(embed_input_)
x = AttentionWithContext()(x)
mlp_input = Input(shape=(params['num_columns'], ))
mlp = BatchNormalization()(mlp_input)
mlp = Dense(256)(mlp)
mlp = PReLU()(mlp)
mlp = BatchNormalization()(mlp)
mlp = Dense(256)(mlp)
mlp = PReLU()(mlp)
merge_layer = concatenate([x, mlp])
x = Dropout(0.1)(merge_layer)
x = Dense(256, activation='relu')(x)
x = Dropout(params['dropout_rate'])(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=[input_, mlp_input], outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
def Conv1DGRUbranched(params):
Embedding_layer = Embedding(params['nb_words'],
params['embedding_dim'],
weights=[params['embedding_matrix']],
input_length=params['sequence_length'],
trainable=False)
input_ = Input(shape=(params['sequence_length'], ))
embed_input_ = Embedding_layer(input_)
conv1 = Conv1D(filters=128,
kernel_size=1,
padding='same',
activation='relu')
conv2 = Conv1D(filters=128,
kernel_size=2,
padding='same',
activation='relu')
conv3 = Conv1D(filters=128,
kernel_size=3,
padding='same',
activation='relu')
conv4 = Conv1D(filters=128,
kernel_size=4,
padding='same',
activation='relu')
conv5 = Conv1D(filters=32,
kernel_size=5,
padding='same',
activation='relu')
conv6 = Conv1D(filters=32,
kernel_size=6,
padding='same',
activation='relu')
conv1a = conv1(embed_input_)
glob1a = GlobalMaxPooling1D()(conv1a)
conv2a = conv2(embed_input_)
glob2a = GlobalMaxPooling1D()(conv2a)
conv3a = conv3(embed_input_)
glob3a = GlobalMaxPooling1D()(conv3a)
conv4a = conv4(embed_input_)
glob4a = GlobalMaxPooling1D()(conv4a)
conv5a = conv5(embed_input_)
glob5a = GlobalMaxPooling1D()(conv5a)
conv6a = conv6(embed_input_)
glob6a = GlobalMaxPooling1D()(conv6a)
if params['bidirectional']:
rnn_branch = Bidirectional(
CuDNNGRU(params['lstm_units'],
return_sequences=True))(embed_input_)
else:
rnn_branch = CuDNNGRU(params['lstm_units'],
return_sequences=True)(embed_input_)
rnn_branch1 = AttentionWithContext()(rnn_branch)
rnn_branch2 = GlobalMaxPooling1D()(rnn_branch)
merge_layer = concatenate([
glob1a, glob2a, glob3a, glob4a, glob5a, glob6a, rnn_branch1,
rnn_branch2
])
x = Dropout(0.2)(merge_layer)
x = BatchNormalization()(x)
x = Dense(256)(x)
x = PReLU()(x)
x = Dropout(0.2)(x)
x = BatchNormalization()(x)
x = Dense(6, activation='sigmoid')(x)
model = Model(inputs=input_, outputs=x)
model.compile(loss=params['loss'],
optimizer=params['optimizer'],
metrics=['accuracy'])
return model
