def GRUattention(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) if params['bidirectional']: x = Bidirectional(CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_) else: x = CuDNNGRU(params['lstm_units'], return_sequences=True)(embed_input_) x = AttentionWithContext()(x) # x = GlobalAveragePooling1D()(x) x = Dropout(params['dropout_rate'])(x) x = Dense(256, activation='relu')(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def GRUHierarchical2(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) l_lstm = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_) l_dense = TimeDistributed(Dense(256))(l_lstm) l_att = GlobalMaxPooling1D()(l_dense) sentEncoder = Model(input_, l_att) review_input = Input(shape=(1, params['sequence_length'])) review_encoder = TimeDistributed(sentEncoder)(review_input) l_lstm_sent = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(review_encoder) l_dense_sent = TimeDistributed(Dense(256))(l_lstm_sent) l_att = GlobalMaxPooling1D()(l_dense_sent) l_att3 = AttentionWithContext()(l_dense_sent) merge_layer2 = concatenate([l_att, l_att3]) x = Dense(6, activation='sigmoid')(merge_layer2) model = Model(inputs=review_input, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def GRUConvdeep3V2(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) x = Activation('tanh')(embed_input_) x = SpatialDropout1D(0.1, name='embed_drop')(x) if params['bidirectional']: x_g1 = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(x) x_l1 = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True))(x_g1) x_g2 = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(x_l1) x1 = GlobalMaxPooling1D()(x_g2) x2 = GlobalAveragePooling1D()(x_g2) x3 = AttentionWithContext()(x_g2) merge_layer = concatenate([x1, x2, x3]) x_conv = Conv1D(64, kernel_size=3, padding='valid', kernel_initializer='glorot_uniform')(x_g2) x1_conv = GlobalMaxPooling1D()(x_conv) x2_conv = GlobalAveragePooling1D()(x_conv) x3_conv = AttentionWithContext()(x_conv) merge_layer2 = concatenate([x1, x2, x3, x1_conv, x2_conv, x3_conv]) x = Dropout(params['dropout_rate'])(merge_layer2) x = Dense(256, activation='relu')(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def LSTMdeep2(params): embed_dropout_rate = 0.1 Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) x = Activation('tanh')(embed_input_) x = SpatialDropout1D(embed_dropout_rate, name='embed_drop')(x) if params['bidirectional']: x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform'))(x) x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform'))(x) x1 = GlobalMaxPooling1D()(x) x2 = GlobalAveragePooling1D()(x) x3 = AttentionWithContext()(x) else: x = CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform')(x) x = CuDNNLSTM(params['lstm_units'], return_sequences=True, kernel_initializer='he_uniform')(x) x = GlobalMaxPooling1D()(x) merge_layer = concatenate([x1, x2, x3]) x = Dropout(params['dropout_rate'])(merge_layer) x = Dense(256)(x) x = PReLU()(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def LSTMattentionBranched(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) if params['bidirectional']: x = Bidirectional( CuDNNLSTM(params['lstm_units'], return_sequences=True))(embed_input_) else: x = CuDNNLSTM(params['lstm_units'], return_sequences=True)(embed_input_) x = AttentionWithContext()(x) mlp_input = Input(shape=(params['num_columns'], )) mlp = BatchNormalization()(mlp_input) mlp = Dense(256)(mlp) mlp = PReLU()(mlp) mlp = BatchNormalization()(mlp) mlp = Dense(256)(mlp) mlp = PReLU()(mlp) merge_layer = concatenate([x, mlp]) x = Dropout(0.1)(merge_layer) x = Dense(256, activation='relu')(x) x = Dropout(params['dropout_rate'])(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=[input_, mlp_input], outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model
def Conv1DGRUbranched(params): Embedding_layer = Embedding(params['nb_words'], params['embedding_dim'], weights=[params['embedding_matrix']], input_length=params['sequence_length'], trainable=False) input_ = Input(shape=(params['sequence_length'], )) embed_input_ = Embedding_layer(input_) conv1 = Conv1D(filters=128, kernel_size=1, padding='same', activation='relu') conv2 = Conv1D(filters=128, kernel_size=2, padding='same', activation='relu') conv3 = Conv1D(filters=128, kernel_size=3, padding='same', activation='relu') conv4 = Conv1D(filters=128, kernel_size=4, padding='same', activation='relu') conv5 = Conv1D(filters=32, kernel_size=5, padding='same', activation='relu') conv6 = Conv1D(filters=32, kernel_size=6, padding='same', activation='relu') conv1a = conv1(embed_input_) glob1a = GlobalMaxPooling1D()(conv1a) conv2a = conv2(embed_input_) glob2a = GlobalMaxPooling1D()(conv2a) conv3a = conv3(embed_input_) glob3a = GlobalMaxPooling1D()(conv3a) conv4a = conv4(embed_input_) glob4a = GlobalMaxPooling1D()(conv4a) conv5a = conv5(embed_input_) glob5a = GlobalMaxPooling1D()(conv5a) conv6a = conv6(embed_input_) glob6a = GlobalMaxPooling1D()(conv6a) if params['bidirectional']: rnn_branch = Bidirectional( CuDNNGRU(params['lstm_units'], return_sequences=True))(embed_input_) else: rnn_branch = CuDNNGRU(params['lstm_units'], return_sequences=True)(embed_input_) rnn_branch1 = AttentionWithContext()(rnn_branch) rnn_branch2 = GlobalMaxPooling1D()(rnn_branch) merge_layer = concatenate([ glob1a, glob2a, glob3a, glob4a, glob5a, glob6a, rnn_branch1, rnn_branch2 ]) x = Dropout(0.2)(merge_layer) x = BatchNormalization()(x) x = Dense(256)(x) x = PReLU()(x) x = Dropout(0.2)(x) x = BatchNormalization()(x) x = Dense(6, activation='sigmoid')(x) model = Model(inputs=input_, outputs=x) model.compile(loss=params['loss'], optimizer=params['optimizer'], metrics=['accuracy']) return model